[llvm-branch-commits] [llvm] [AMDGPU] Update calling convention for true16 mode (PR #218463)
Guo Chen via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 15 14:05:02 PDT 2026
https://github.com/broxigarchen updated https://github.com/llvm/llvm-project/pull/218463
>From 6c61bb4c8a2e41b2efd3f21cebf8ea99d7fc4d01 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Wed, 1 Apr 2026 11:20:27 -0400
Subject: [PATCH 1/3] calling conv update
---
llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp | 27 +-
llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td | 69 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 5 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll | 27 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll | 30 +-
.../CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll | 19 +-
.../AMDGPU/GlobalISel/fcanonicalize.bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll | 2 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll | 2 +-
.../CodeGen/AMDGPU/GlobalISel/fma.bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll | 2 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll | 2 +-
.../CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll | 19 +-
.../CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll | 441 +-
.../test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll | 527 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 32 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 36 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll | 19 +-
.../AMDGPU/GlobalISel/insertelement.i16.ll | 141 +-
.../AMDGPU/GlobalISel/intrinsic-trunc.ll | 2 +-
...galize-llvm.amdgcn.image.atomic.dim.a16.ll | 798 +-
.../legalize-llvm.amdgcn.image.sample.a16.ll | 2176 ++--
...galize-llvm.amdgcn.image.sample.g16.a16.ll | 204 +-
.../legalize-llvm.amdgcn.image.sample.g16.ll | 1048 +-
...legalize-llvm.amdgcn.image.store.2d.d16.ll | 8 +-
.../AMDGPU/GlobalISel/llvm.amdgcn.fract.ll | 2 +-
.../llvm.amdgcn.image.atomic.dim.a16.ll | 40 +-
.../llvm.amdgcn.image.gather4.a16.dim.ll | 135 +-
.../llvm.amdgcn.image.load.2darraymsaa.a16.ll | 25 +-
.../llvm.amdgcn.image.load.3d.a16.ll | 19 +-
.../llvm.amdgcn.image.sample.g16.ll | 18 +-
.../GlobalISel/llvm.amdgcn.interp.inreg.ll | 2 +-
.../llvm.amdgcn.make.buffer.rsrc.ll | 26 +-
.../llvm.amdgcn.raw.buffer.load.format.f16.ll | 16 +-
.../GlobalISel/llvm.amdgcn.raw.buffer.load.ll | 15 +-
.../llvm.amdgcn.raw.ptr.buffer.load.ll | 10 +-
.../llvm.amdgcn.raw.tbuffer.load.f16.ll | 36 +-
.../llvm.amdgcn.raw.tbuffer.store.i8.ll | 50 +-
.../llvm.amdgcn.struct.buffer.load.ll | 10 +-
...llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll | 52 -
.../llvm.amdgcn.struct.tbuffer.load.f16.ll | 67 +-
.../AMDGPU/GlobalISel/load-constant.96.ll | 22 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll | 27 +-
.../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 6 +-
.../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 6 +-
.../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll | 6 +-
.../test/CodeGen/AMDGPU/GlobalISel/usubsat.ll | 6 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll | 27 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 4071 +++----
.../CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll | 352 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll | 1547 ++-
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 2918 +++--
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 2455 ++---
.../arbitrary-fp-from-float-fp8-f16-hw.ll | 8 +-
.../AMDGPU/arbitrary-fp-to-float-fp8-hw.ll | 102 +-
llvm/test/CodeGen/AMDGPU/bf16.ll | 9518 ++++++++---------
llvm/test/CodeGen/AMDGPU/bitop3.ll | 81 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 642 +-
.../buffer-fat-pointer-atomicrmw-fmax.ll | 642 +-
.../buffer-fat-pointer-atomicrmw-fmin.ll | 642 +-
.../CodeGen/AMDGPU/call-argument-types.ll | 45 +-
.../CodeGen/AMDGPU/calling-conventions.ll | 448 +-
.../CodeGen/AMDGPU/combine_andor_with_cmps.ll | 6 +-
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll | 25 +-
.../CodeGen/AMDGPU/dagcombine-fmul-sel.ll | 204 +-
.../AMDGPU/divergence-driven-buildvector.ll | 5 +-
.../test/CodeGen/AMDGPU/dynamic_stackalloc.ll | 2 +-
.../test/CodeGen/AMDGPU/fcanonicalize.bf16.ll | 11 +-
llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 297 +-
llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll | 6 +-
llvm/test/CodeGen/AMDGPU/fdiv.f16.ll | 8 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 1659 ++-
.../CodeGen/AMDGPU/flat-atomicrmw-fmax.ll | 2859 +++--
.../CodeGen/AMDGPU/flat-atomicrmw-fmin.ll | 2859 +++--
.../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll | 1637 ++-
llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll | 165 +-
.../float-to-arbitrary-fp-fp8-f16-hw.ll | 36 +-
.../AMDGPU/float-to-arbitrary-fp-fp8-hw.ll | 220 +-
.../AMDGPU/float-to-arbitrary-fp-widen.ll | 2844 +++--
llvm/test/CodeGen/AMDGPU/fma.f16.ll | 26 +-
llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll | 272 +-
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 36 +-
llvm/test/CodeGen/AMDGPU/fmaximum3.ll | 8 +-
llvm/test/CodeGen/AMDGPU/fmaxnum.ll | 235 +-
llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll | 26 +-
llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll | 272 +-
llvm/test/CodeGen/AMDGPU/fminimum.ll | 36 +-
llvm/test/CodeGen/AMDGPU/fminimum3.ll | 8 +-
llvm/test/CodeGen/AMDGPU/fminnum.ll | 218 +-
llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll | 32 +-
.../CodeGen/AMDGPU/fneg-modifier-casting.ll | 72 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 1086 +-
llvm/test/CodeGen/AMDGPU/function-args.ll | 62 +-
llvm/test/CodeGen/AMDGPU/function-returns.ll | 45 +-
.../AMDGPU/gfx-callable-argument-types.ll | 554 +-
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 2162 ++--
.../CodeGen/AMDGPU/global-atomicrmw-fmax.ll | 2768 +++--
.../CodeGen/AMDGPU/global-atomicrmw-fmin.ll | 2768 +++--
.../CodeGen/AMDGPU/global-atomicrmw-fsub.ll | 1637 ++-
llvm/test/CodeGen/AMDGPU/global-saddr-load.ll | 32 +-
llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll | 710 +-
.../CodeGen/AMDGPU/integer-mad-patterns.ll | 193 +-
.../legalize-amdgcn.raw.ptr.buffer.load.ll | 8 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll | 1 +
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll | 2 +-
.../llvm.amdgcn.image.gather4.a16.dim.ll | 39 +-
.../AMDGPU/llvm.amdgcn.image.msaa.load.ll | 16 +-
.../llvm.amdgcn.image.sample.a16.dim.ll | 131 +-
.../llvm.amdgcn.image.sample.d16.dim.ll | 58 +-
.../llvm.amdgcn.image.sample.g16.encode.ll | 16 +-
.../AMDGPU/llvm.amdgcn.image.sample.g16.ll | 6 +-
.../AMDGPU/llvm.amdgcn.interp.inreg.ll | 2 +-
.../AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll | 4 +-
.../llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll | 18 +-
.../llvm.amdgcn.raw.tbuffer.load.d16.ll | 36 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll | 110 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll | 110 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll | 110 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll | 110 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll | 110 +-
...llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll | 20 +-
.../llvm.amdgcn.struct.tbuffer.load.d16.ll | 40 +-
.../AMDGPU/llvm.amdgcn.waitcnt.out.order.ll | 28 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll | 41 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.f16.ll | 73 +-
llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll | 497 +-
llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll | 8 +-
llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll | 8 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/lround.ll | 26 +-
llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll | 31 +-
llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll | 36 +-
llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll | 4966 +++++----
llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll | 4966 +++++----
.../CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll | 3 +-
.../AMDGPU/pseudo-scalar-transcendental.ll | 342 +-
.../AMDGPU/reassoc-mul-add-1-to-mad.ll | 40 +-
llvm/test/CodeGen/AMDGPU/repeated-divisor.ll | 14 +-
llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll | 42 +-
llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll | 209 +-
llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll | 16 +-
llvm/test/CodeGen/AMDGPU/scmp.ll | 22 +-
.../AMDGPU/select-fabs-fneg-extract.f16.ll | 54 +-
llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll | 4 +-
llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll | 4 +-
llvm/test/CodeGen/AMDGPU/usubsat.ll | 9 +-
llvm/test/CodeGen/AMDGPU/v_swap_b16.ll | 46 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll | 44 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll | 44 +-
.../test/CodeGen/AMDGPU/vector-reduce-fadd.ll | 140 +-
.../test/CodeGen/AMDGPU/vector-reduce-fmul.ll | 140 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll | 44 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll | 44 +-
.../test/CodeGen/AMDGPU/vector-reduce-smax.ll | 234 +-
.../test/CodeGen/AMDGPU/vector-reduce-smin.ll | 234 +-
.../test/CodeGen/AMDGPU/vector-reduce-umax.ll | 164 +-
.../test/CodeGen/AMDGPU/vector-reduce-umin.ll | 164 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll | 44 +-
.../test/CodeGen/AMDGPU/vector_rebroadcast.ll | 124 +-
164 files changed, 36954 insertions(+), 32761 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
index 2a78237cf7e20..63a342aacf036 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp
@@ -32,10 +32,14 @@ namespace {
/// Wrapper around extendRegister to ensure we extend to a full 32-bit register.
static Register extendRegisterMin32(CallLowering::ValueHandler &Handler,
- Register ValVReg, const CCValAssign &VA) {
- if (VA.getLocVT().getSizeInBits() < 32) {
+ Register ValVReg, const CCValAssign &VA,
+ bool isPhyRegSGPR, const GCNSubtarget &ST) {
+ if (VA.getLocVT().getSizeInBits() < 32 &&
+ (isPhyRegSGPR || !ST.useRealTrue16Insts())) {
// 16-bit types are reported as legal for 32-bit registers. We need to
// extend and do a 32-bit copy to avoid the verifier complaining about it.
+ //
+ // In true16 mode, we still extend to 32 if the physical reg is a sgpr
return Handler.MIRBuilder.buildAnyExt(LLT::integer(32), ValVReg).getReg(0);
}
@@ -64,14 +68,17 @@ struct AMDGPUOutgoingValueHandler : public CallLowering::OutgoingValueHandler {
void assignValueToReg(Register ValVReg, Register PhysReg,
const CCValAssign &VA,
ISD::ArgFlagsTy Flags = {}) override {
- Register ExtReg = extendRegisterMin32(*this, ValVReg, VA);
+ const SIRegisterInfo *TRI =
+ static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
+ const GCNSubtarget &ST = MIRBuilder.getMF().getSubtarget<GCNSubtarget>();
+ bool isPhyRegSGPR = TRI->isSGPRReg(MRI, PhysReg);
+
+ Register ExtReg = extendRegisterMin32(*this, ValVReg, VA, isPhyRegSGPR, ST);
// If this is a scalar return, insert a readfirstlane just in case the value
// ends up in a VGPR.
// FIXME: Assert this is a shader return.
- const SIRegisterInfo *TRI
- = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
- if (TRI->isSGPRReg(MRI, PhysReg)) {
+ if (isPhyRegSGPR) {
LLT Ty = MRI.getType(ExtReg);
LLT I32 = LLT::integer(32);
if (Ty != I32 && Ty != LLT::float32()) {
@@ -119,7 +126,13 @@ struct AMDGPUIncomingArgHandler : public CallLowering::IncomingValueHandler {
}
void copyToReg(Register ValVReg, Register PhysReg, const CCValAssign &VA) {
- if (VA.getLocVT().getSizeInBits() < 32) {
+ const GCNSubtarget &ST = MIRBuilder.getMF().getSubtarget<GCNSubtarget>();
+ const SIRegisterInfo *TRI =
+ static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo());
+ bool isPhyRegSGPR = TRI->isSGPRReg(MRI, PhysReg);
+
+ if (VA.getLocVT().getSizeInBits() < 32 &&
+ (isPhyRegSGPR || !ST.useRealTrue16Insts())) {
// 16-bit types are reported as legal for 32-bit registers. We need to
// do a 32-bit copy, and truncate to avoid the verifier complaining
// about it.
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
index 2932bbf0e7bbd..51740440c86d7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCallingConv.td
@@ -18,6 +18,10 @@ class CCIfOrigTypeShaderCCIsSGPR<CCAction A>
: CCIf<[{(!OrigTy->getScalarType()->isFloatTy() &&
!OrigTy->getScalarType()->isHalfTy()) }], A>;
+class CCIfTrue16<CCAction A> :
+ CCIf<"State.getMachineFunction().getSubtarget<GCNSubtarget>().useRealTrue16Insts()", A>;
+class CCIfNotTrue16<CCAction A> :
+ CCIf<"!State.getMachineFunction().getSubtarget<GCNSubtarget>().useRealTrue16Insts()", A>;
// Calling convention for SI
def CC_SI_Gfx : CallingConv<[
@@ -30,10 +34,18 @@ def CC_SI_Gfx : CallingConv<[
!foreach(i, !range(4, 30), !cast<Register>("SGPR"#i)) // SGPR4-29
>>>,
- CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+ CCIfNotInReg<CCIfType<[f32, i32, v2i16, v2f16, v2bf16] , CCAssignToReg<
!foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
>>>,
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-31_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
+ >>>,
+
CCIfType<[i32, f32, v2i16, v2f16, i16, f16, i1, bf16, v2bf16], CCAssignToStack<4, 4>>
]>;
@@ -41,9 +53,17 @@ def RetCC_SI_Gfx : CallingConv<[
CCIfType<[i1], CCPromoteToType<i32>>,
CCIfType<[i1, i16], CCIfExtend<CCPromoteToType<i32>>>,
- CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+ CCIfNotInReg<CCIfType<[f32, i32, v2i16, v2f16, v2bf16] , CCAssignToReg<
!foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
>>>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-31_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
+ >>>,
]>;
def CC_SI_SHADER : CallingConv<[
@@ -55,9 +75,17 @@ def CC_SI_SHADER : CallingConv<[
>>>,
// 32*4 + 4 is the minimum for a fetch shader consumer with 32 inputs.
- CCIfNotInReg<CCIfType<[f32, i32, f16, i16, v2i16, v2f16, bf16, v2bf16] , CCAssignToReg<
+ CCIfNotInReg<CCIfType<[f32, i32, v2i16, v2f16, v2bf16] , CCAssignToReg<
!foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
- >>>
+ >>>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-135_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
+ >>>,
]>;
@@ -68,9 +96,17 @@ def RetCC_SI_Shader : CallingConv<[
>>>,
// 32*4 + 4 is the minimum for a fetch shader with 32 outputs.
- CCIfType<[f32, f16, v2f16, bf16, v2bf16, i32, i16, v2i16] , CCAssignToReg<
+ CCIfType<[f32, v2f16, v2bf16, i32, v2i16] , CCAssignToReg<
!foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
- >>
+ >>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-135_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 136), !cast<Register>("VGPR"#i)) // VGPR0-135
+ >>>,
]>;
def CSR_AMDGPU_VGPRs : CalleeSavedRegs<
@@ -143,9 +179,18 @@ def CC_AMDGPU_Func : CallingConv<[
!foreach(i, !range(0, 30), !cast<Register>("SGPR"#i)) // SGPR0-29
>>>,
- CCIfType<[i32, f32, i16, f16, v2i16, v2f16, i1, bf16, v2bf16], CCAssignToReg<
+ CCIfType<[i32, f32, v2i16, v2f16, i1, v2bf16], CCAssignToReg<
!foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
>>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-31_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
+ >>>,
+
CCIfType<[i32, f32, v2i16, v2f16, i16, f16, i1, bf16, v2bf16], CCAssignToStack<4, 4>>
]>;
@@ -153,9 +198,17 @@ def CC_AMDGPU_Func : CallingConv<[
def RetCC_AMDGPU_Func : CallingConv<[
CCIfType<[i1], CCPromoteToType<i32>>,
CCIfType<[i1, i16], CCIfExtend<CCPromoteToType<i32>>>,
- CCIfType<[i32, f32, i16, f16, v2i16, v2f16, bf16, v2bf16], CCAssignToReg<
+ CCIfType<[i32, f32, v2i16, v2f16, v2bf16], CCAssignToReg<
!foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
>>,
+
+ CCIfTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i#"_LO16")) // VGPR0-31_LO16
+ >>>,
+
+ CCIfNotTrue16<CCIfType<[i16, f16, bf16], CCAssignToReg<
+ !foreach(i, !range(0, 32), !cast<Register>("VGPR"#i)) // VGPR0-31
+ >>>,
]>;
def CC_AMDGPU : CallingConv<[
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index e6186c77d9268..eb32d88c4af41 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -3804,12 +3804,15 @@ SDValue SITargetLowering::LowerFormalArguments(
RC = &AMDGPU::VGPR_32RegClass;
else if (AMDGPU::SGPR_32RegClass.contains(Reg))
RC = &AMDGPU::SGPR_32RegClass;
+ else if (AMDGPU::VGPR_16RegClass.contains(Reg))
+ RC = &AMDGPU::VGPR_16RegClass;
else
llvm_unreachable("Unexpected register class in LowerFormalArguments!");
Reg = MF.addLiveIn(Reg, RC);
SDValue Val = DAG.getCopyFromReg(Chain, DL, Reg, VT);
- if (Arg.Flags.isInReg() && RC == &AMDGPU::VGPR_32RegClass) {
+ if (Arg.Flags.isInReg() &&
+ (RC == &AMDGPU::VGPR_32RegClass || RC == &AMDGPU::VGPR_16RegClass)) {
// FIXME: Need to forward the chains created by `CopyFromReg`s, make sure
// they will read physical regs before any side effect instructions.
SDValue ReadFirstLane =
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll
index ddc1f68e8fbe0..4eb06b667b238 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/and.ll
@@ -33,11 +33,23 @@ define i16 @v_and_i16(i16 %num, i16 %den) {
; GCN-NEXT: v_and_b32_e32 v0, v0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_and_i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_and_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_and_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_and_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: v_and_i16:
; GFX12: ; %bb.0:
@@ -46,7 +58,7 @@ define i16 @v_and_i16(i16 %num, i16 %den) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX12-NEXT: v_and_b16 v0.l, v0.l, v1.l
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = and i16 %num, %den
ret i16 %result
@@ -1134,6 +1146,3 @@ define <2 x i256> @v_and_v2i256(<2 x i256> %a, <2 x i256> %b) {
ret <2 x i256> %and
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-FAKE16: {{.*}}
-; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index 8a4d6c1d67b4d..a4a5665aba28d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -38,9 +38,9 @@ define i8 @v_ashr_i8(i8 %value, i8 %amount) {
; GFX11-TRUE16-LABEL: v_ashr_i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_ashr_i8:
@@ -2055,18 +2055,18 @@ define <4 x i2> @v_ashr_v4i2(<4 x i2> %value, <4 x i2> %amount) {
; GFX11-TRUE16-LABEL: v_ashr_v4i2:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 2
-; GFX11-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 2
-; GFX11-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 2
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 3
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v5.l, 3
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v6.l, 3
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v7.l, 3
-; GFX11-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 2
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v1.l, v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v2.l, v1.h, v2.l
-; GFX11-TRUE16-NEXT: v_ashrrev_i16 v3.l, v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v4.l, 3
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v5.l, 3
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v6.l, 3
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v7.l, 3
+; GFX11-TRUE16-NEXT: v_bfe_i32 v4, v0, 0, 2
+; GFX11-TRUE16-NEXT: v_bfe_i32 v5, v1, 0, 2
+; GFX11-TRUE16-NEXT: v_bfe_i32 v6, v2, 0, 2
+; GFX11-TRUE16-NEXT: v_bfe_i32 v7, v3, 0, 2
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v0.l, v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v1.l, v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v2.l, v2.h, v6.l
+; GFX11-TRUE16-NEXT: v_ashrrev_i16 v3.l, v3.h, v7.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_ashr_v4i2:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
index 5b6edff208fd4..216f28c6fd55f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.bf16.ll
@@ -81,7 +81,7 @@ define amdgpu_ps bfloat @fadd_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX12-NEXT: s_cmp_u_f32 s0, 0
; GFX12-NEXT: s_cselect_b32 s0, s2, s1
; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: fadd_bf16_ss:
@@ -100,7 +100,7 @@ define amdgpu_ps bfloat @fadd_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX1250-NEXT: s_cmp_u_f32 s0, 0
; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fadd bfloat %a, %b
ret bfloat %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
index bdeacefca362f..e4d707106ea13 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fadd.ll
@@ -15,12 +15,19 @@ define amdgpu_ps half @fadd_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fadd_s16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_add_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fadd_s16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_add_f16 s0, s0, s1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fadd_s16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_add_f16 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
%fadd = fadd half %a, %b
ret half %fadd
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
index 086d7e295c006..367e40fa0bad0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fcanonicalize.bf16.ll
@@ -82,7 +82,7 @@ define amdgpu_ps bfloat @fcanonicalize_bf16_s(bfloat inreg %src) {
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_lshr_b32 s0, s0, 16
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: fcanonicalize_bf16_s:
@@ -101,7 +101,7 @@ define amdgpu_ps bfloat @fcanonicalize_bf16_s(bfloat inreg %src) {
; GFX1250-NEXT: s_cmp_u_f32 s0, 0
; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.canonicalize.bf16(bfloat %src)
ret bfloat %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
index a25a30fa5a8a3..be230801dd17b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fceil.ll
@@ -62,7 +62,7 @@ define half @fceil_s16_s(half inreg %val) {
; GFX12-NEXT: s_ceil_f16 s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.ceil.f16(half %val)
ret half %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
index c1785c0f62b12..8bff476594d26 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ffloor.ll
@@ -62,7 +62,7 @@ define half @ffloor_s16_s(half inreg %val) {
; GFX12-NEXT: s_floor_f16 s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.floor.f16(half %val)
ret half %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
index f411349cf92f8..60ce2c26a1477 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.bf16.ll
@@ -87,7 +87,7 @@ define amdgpu_ps bfloat @fma_bf16_sss(bfloat inreg %a, bfloat inreg %b, bfloat i
; GFX12-NEXT: s_cmp_u_f32 s2, 0
; GFX12-NEXT: s_cselect_b32 s0, s1, s0
; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: fma_bf16_sss:
@@ -107,7 +107,7 @@ define amdgpu_ps bfloat @fma_bf16_sss(bfloat inreg %a, bfloat inreg %b, bfloat i
; GFX1250-NEXT: s_cmp_u_f32 s2, 0
; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
ret bfloat %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
index 639469de762b4..3b5347c699581 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
@@ -1539,7 +1539,7 @@ define amdgpu_ps half @fma_s16_uniform(half inreg %a, half inreg %b, half inreg
; GFX12-LABEL: fma_s16_uniform:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_fmac_f16 s2, s0, s1
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-NEXT: ; return to shader part epilog
%fma = call half @llvm.fma.f16(half %a, half %b, half %c)
ret half %fma
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
index 0baba45e541a1..13a4cb26dc978 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3.ll
@@ -38,7 +38,7 @@ define amdgpu_ps half @fmed3_s16_uniform_salu_use(half inreg %a, half inreg %b,
; GFX12-NEXT: s_add_f16 s0, s0, s3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
%result = call half @llvm.amdgcn.fmed3.f16(half %a, half %b, half %c)
%add = fadd half %result, %d
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
index 9c02c096aaf4c..a4e47401d0bb4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.bf16.ll
@@ -81,7 +81,7 @@ define amdgpu_ps bfloat @fmul_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX12-NEXT: s_cmp_u_f32 s0, 0
; GFX12-NEXT: s_cselect_b32 s0, s2, s1
; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
;
; GFX1250-LABEL: fmul_bf16_ss:
@@ -100,7 +100,7 @@ define amdgpu_ps bfloat @fmul_bf16_ss(bfloat inreg %a, bfloat inreg %b) {
; GFX1250-NEXT: s_cmp_u_f32 s0, 0
; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, s0
; GFX1250-NEXT: ; return to shader part epilog
%result = fmul bfloat %a, %b
ret bfloat %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
index f4fb2d4595236..392359a2f7323 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.ll
@@ -15,12 +15,19 @@ define amdgpu_ps half @fmul_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-TRUE16-NEXT: v_mul_f16_e64 v0.l, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fmul_s16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mul_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fmul_s16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_mul_f16 s0, s0, s1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fmul_s16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_mul_f16 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
%result = fmul half %a, %b
ret half %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
index f9e9b145aa955..d94b72402dab6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
@@ -7,51 +7,97 @@
; RUN: llc -global-isel -mtriple=amdgpu12.50-amd-amdpal -mattr=+real-true16 -o - %s | FileCheck -check-prefixes=GFX1250,GFX1250-TRUE16 %s
define amdgpu_ps bfloat @fptrunc_f32_to_bf16_s(float inreg %a) {
-; GFX11-LABEL: fptrunc_f32_to_bf16_s:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX11-NEXT: v_cmp_u_f32_e64 s2, s0, 0
-; GFX11-NEXT: s_add_i32 s1, s1, s0
-; GFX11-NEXT: s_bitset1_b32 s0, 22
-; GFX11-NEXT: s_addk_i32 s1, 0x7fff
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s2, s0, 0
+; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, s0
+; GFX11-FAKE16-NEXT: s_bitset1_b32 s0, 22
+; GFX11-FAKE16-NEXT: s_addk_i32 s1, 0x7fff
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, s0, 0
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, s0
+; GFX11-TRUE16-NEXT: s_bitset1_b32 s0, 22
+; GFX11-TRUE16-NEXT: s_addk_i32 s1, 0x7fff
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-FAKE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-FAKE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-FAKE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-TRUE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-TRUE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-TRUE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fptrunc_f32_to_bf16_s:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX12-NEXT: s_add_co_i32 s1, s1, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX12-NEXT: s_cmp_u_f32 s0, 0
-; GFX12-NEXT: s_cselect_b32 s0, s2, s1
-; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX1250-FAKE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-FAKE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-FAKE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-FAKE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: fptrunc_f32_to_bf16_s:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX1250-TRUE16-LABEL: fptrunc_f32_to_bf16_s:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-TRUE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-TRUE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-TRUE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
%result = fptrunc float %a to bfloat
ret bfloat %result
}
@@ -141,110 +187,215 @@ define amdgpu_ps bfloat @fptrunc_f32_to_bf16_v(float %a) {
}
define amdgpu_ps bfloat @fptrunc_f64_to_bf16_s(double inreg %a) {
-; GFX11-LABEL: fptrunc_f64_to_bf16_s:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
-; GFX11-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX11-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
-; GFX11-NEXT: v_readfirstlane_b32 s1, v2
-; GFX11-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; GFX11-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s2, s2, s1
-; GFX11-NEXT: s_cmp_lg_u32 s0, 0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_and_b32 s2, s2, 1
-; GFX11-NEXT: s_or_b32 s0, s0, 1
-; GFX11-NEXT: s_add_i32 s0, s1, s0
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s0, s1, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX11-NEXT: v_cmp_u_f32_e64 s2, s0, 0
-; GFX11-NEXT: s_add_i32 s1, s1, s0
-; GFX11-NEXT: s_bitset1_b32 s0, 22
-; GFX11-NEXT: s_addk_i32 s1, 0x7fff
-; GFX11-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s0, s0, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX11-FAKE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX11-FAKE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s2, s1
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 1
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 1
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e64 s2, s0, 0
+; GFX11-FAKE16-NEXT: s_add_i32 s1, s1, s0
+; GFX11-FAKE16-NEXT: s_bitset1_b32 s0, 22
+; GFX11-FAKE16-NEXT: s_addk_i32 s1, 0x7fff
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fptrunc_f64_to_bf16_s:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
-; GFX12-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX12-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
-; GFX12-NEXT: v_readfirstlane_b32 s1, v2
-; GFX12-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; GFX12-NEXT: s_cselect_b32 s2, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s2, s2, s1
-; GFX12-NEXT: s_cmp_lg_u32 s0, 0
-; GFX12-NEXT: s_cselect_b32 s0, -1, 0
-; GFX12-NEXT: s_and_b32 s2, s2, 1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, s0, 1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_i32 s0, s1, s0
-; GFX12-NEXT: s_cmp_lg_u32 s2, 0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_cselect_b32 s0, s1, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX12-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_i32 s1, s1, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX12-NEXT: s_cmp_u_f32 s0, 0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_cselect_b32 s0, s2, s1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshr_b32 s0, s0, 16
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX11-TRUE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX11-TRUE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s2, s2, s1
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, 1
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, s0, 0
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, s0
+; GFX11-TRUE16-NEXT: s_bitset1_b32 s0, 22
+; GFX11-TRUE16-NEXT: s_addk_i32 s1, 0x7fff
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: fptrunc_f64_to_bf16_s:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
-; GFX1250-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
-; GFX1250-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
-; GFX1250-NEXT: v_readfirstlane_b32 s1, v2
-; GFX1250-NEXT: s_cmp_lg_u32 vcc_lo, 0
-; GFX1250-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_or_b32 s2, s2, s1
-; GFX1250-NEXT: s_cmp_lg_u32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, -1, 0
-; GFX1250-NEXT: s_and_b32 s2, s2, 1
-; GFX1250-NEXT: s_or_b32 s0, s0, 1
-; GFX1250-NEXT: s_add_co_i32 s0, s1, s0
-; GFX1250-NEXT: s_cmp_lg_u32 s2, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_bfe_u32 s1, s0, 0x10010
-; GFX1250-NEXT: s_or_b32 s2, s0, 0x400000
-; GFX1250-NEXT: s_add_co_i32 s1, s1, s0
-; GFX1250-NEXT: s_addk_co_i32 s1, 0x7fff
-; GFX1250-NEXT: s_cmp_u_f32 s0, 0
-; GFX1250-NEXT: s_cselect_b32 s0, s2, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 16
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX12-FAKE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX12-FAKE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s2, s1
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, 1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s0, 1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_add_co_i32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-FAKE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-FAKE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX12-TRUE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX12-TRUE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s2, s2, s1
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s0, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_add_co_i32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX12-TRUE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX12-TRUE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-FAKE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX1250-FAKE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX1250-FAKE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX1250-FAKE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_or_b32 s2, s2, s1
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-FAKE16-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s0, 1
+; GFX1250-FAKE16-NEXT: s_add_co_i32 s0, s1, s0
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-FAKE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-FAKE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-FAKE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-FAKE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-TRUE16-LABEL: fptrunc_f64_to_bf16_s:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: v_cvt_f32_f64_e32 v2, s[0:1]
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
+; GFX1250-TRUE16-NEXT: v_cmp_nlg_f64_e32 vcc_lo, s[0:1], v[0:1]
+; GFX1250-TRUE16-NEXT: v_cmp_ngt_f64_e64 s0, |s[0:1]|, |v[0:1]|
+; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s1, v2
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 vcc_lo, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_or_b32 s2, s2, s1
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 s0, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
+; GFX1250-TRUE16-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s0, 1
+; GFX1250-TRUE16-NEXT: s_add_co_i32 s0, s1, s0
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s1, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s1, s0, 0x10010
+; GFX1250-TRUE16-NEXT: s_or_b32 s2, s0, 0x400000
+; GFX1250-TRUE16-NEXT: s_add_co_i32 s1, s1, s0
+; GFX1250-TRUE16-NEXT: s_addk_co_i32 s1, 0x7fff
+; GFX1250-TRUE16-NEXT: s_cmp_u_f32 s0, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s2, s1
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
%result = fptrunc double %a to bfloat
ret bfloat %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
index d34b222eb56ea..8b251d1beb541 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.ll
@@ -17,24 +17,43 @@ define amdgpu_ps half @fptrunc_f32_to_f16_uniform(float inreg %a) {
; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fptrunc_f32_to_f16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_cvt_f16_f32 s0, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fptrunc_f32_to_f16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_cvt_f16_f32 s0, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: fptrunc_f32_to_f16_uniform:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_cvt_f16_f32 s0, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX12-TRUE16-LABEL: fptrunc_f32_to_f16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_cvt_f16_f32 s0, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-FAKE16-LABEL: fptrunc_f32_to_f16_uniform:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_cvt_f16_f32 s0, s0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-TRUE16-LABEL: fptrunc_f32_to_f16_uniform:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_cvt_f16_f32 s0, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
%result = fptrunc float %a to half
ret half %result
}
@@ -130,165 +149,325 @@ define amdgpu_ps float @fptrunc_f64_to_f32_div(double %a) {
}
define amdgpu_ps half @fptrunc_f64_to_f16_uniform(double inreg %a) {
-; GFX11-LABEL: fptrunc_f64_to_f16_uniform:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_bfe_u32 s2, s1, 0xb0014
-; GFX11-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-NEXT: s_and_b32 s4, s1, 0x1ff
-; GFX11-NEXT: s_addk_i32 s2, 0xfc10
-; GFX11-NEXT: s_and_b32 s3, s3, 0xffe
-; GFX11-NEXT: s_or_b32 s0, s4, s0
-; GFX11-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s3, s0
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_sub_i32 s4, 1, s2
-; GFX11-NEXT: s_or_b32 s5, s0, 0x1000
-; GFX11-NEXT: s_max_i32 s4, s4, 0
-; GFX11-NEXT: s_lshl_b32 s3, s3, 9
-; GFX11-NEXT: s_min_i32 s4, s4, 13
-; GFX11-NEXT: s_lshl_b32 s7, s2, 12
-; GFX11-NEXT: s_lshr_b32 s6, s5, s4
-; GFX11-NEXT: s_or_b32 s3, s3, 0x7c00
-; GFX11-NEXT: s_lshl_b32 s4, s6, s4
-; GFX11-NEXT: s_or_b32 s0, s0, s7
-; GFX11-NEXT: s_cmp_lg_u32 s4, s5
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s4, s6, s4
-; GFX11-NEXT: s_cmp_lt_i32 s2, 1
-; GFX11-NEXT: s_cselect_b32 s0, s4, s0
-; GFX11-NEXT: s_and_b32 s4, s0, 7
-; GFX11-NEXT: s_lshr_b32 s0, s0, 2
-; GFX11-NEXT: s_cmp_eq_u32 s4, 3
-; GFX11-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-NEXT: s_cmp_gt_i32 s4, 5
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s4, s5, s4
-; GFX11-NEXT: s_cmp_lg_u32 s4, 0
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_i32 s0, s0, s4
-; GFX11-NEXT: s_cmp_gt_i32 s2, 30
-; GFX11-NEXT: s_cselect_b32 s0, 0x7c00, s0
-; GFX11-NEXT: s_cmpk_eq_i32 s2, 0x40f
-; GFX11-NEXT: s_cselect_b32 s0, s3, s0
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s1, s1, 0x8000
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-FAKE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX11-FAKE16-NEXT: s_addk_i32 s2, 0xfc10
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s4, s0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s3, s0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-FAKE16-NEXT: s_sub_i32 s4, 1, s2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX11-FAKE16-NEXT: s_max_i32 s4, s4, 0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX11-FAKE16-NEXT: s_min_i32 s4, s4, 13
+; GFX11-FAKE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX11-FAKE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s7
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s6, s4
+; GFX11-FAKE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s0, 7
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX11-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s4, s5, s4
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX11-FAKE16-NEXT: s_cmpk_eq_i32 s2, 0x40f
+; GFX11-FAKE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fptrunc_f64_to_f16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_bfe_u32 s2, s1, 0xb0014
-; GFX12-NEXT: s_lshr_b32 s3, s1, 8
-; GFX12-NEXT: s_and_b32 s4, s1, 0x1ff
-; GFX12-NEXT: s_addk_co_i32 s2, 0xfc10
-; GFX12-NEXT: s_and_b32 s3, s3, 0xffe
-; GFX12-NEXT: s_or_b32 s0, s4, s0
-; GFX12-NEXT: s_cselect_b32 s0, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s0, s3, s0
-; GFX12-NEXT: s_cselect_b32 s3, 1, 0
-; GFX12-NEXT: s_sub_co_i32 s4, 1, s2
-; GFX12-NEXT: s_or_b32 s5, s0, 0x1000
-; GFX12-NEXT: s_max_i32 s4, s4, 0
-; GFX12-NEXT: s_lshl_b32 s3, s3, 9
-; GFX12-NEXT: s_min_i32 s4, s4, 13
-; GFX12-NEXT: s_lshl_b32 s7, s2, 12
-; GFX12-NEXT: s_lshr_b32 s6, s5, s4
-; GFX12-NEXT: s_or_b32 s3, s3, 0x7c00
-; GFX12-NEXT: s_lshl_b32 s4, s6, s4
-; GFX12-NEXT: s_or_b32 s0, s0, s7
-; GFX12-NEXT: s_cmp_lg_u32 s4, s5
-; GFX12-NEXT: s_cselect_b32 s4, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s4, s6, s4
-; GFX12-NEXT: s_cmp_lt_i32 s2, 1
-; GFX12-NEXT: s_cselect_b32 s0, s4, s0
-; GFX12-NEXT: s_and_b32 s4, s0, 7
-; GFX12-NEXT: s_lshr_b32 s0, s0, 2
-; GFX12-NEXT: s_cmp_eq_u32 s4, 3
-; GFX12-NEXT: s_cselect_b32 s5, 1, 0
-; GFX12-NEXT: s_cmp_gt_i32 s4, 5
-; GFX12-NEXT: s_cselect_b32 s4, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s4, s5, s4
-; GFX12-NEXT: s_cmp_lg_u32 s4, 0
-; GFX12-NEXT: s_cselect_b32 s4, 1, 0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_add_co_i32 s0, s0, s4
-; GFX12-NEXT: s_cmp_gt_i32 s2, 30
-; GFX12-NEXT: s_cselect_b32 s0, 0x7c00, s0
-; GFX12-NEXT: s_cmp_eq_u32 s2, 0x40f
-; GFX12-NEXT: s_cselect_b32 s0, s3, s0
-; GFX12-NEXT: s_lshr_b32 s1, s1, 16
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_b32 s1, s1, 0x8000
-; GFX12-NEXT: s_or_b32 s0, s1, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX11-TRUE16-NEXT: s_addk_i32 s2, 0xfc10
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s4, s0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-TRUE16-NEXT: s_sub_i32 s4, 1, s2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX11-TRUE16-NEXT: s_max_i32 s4, s4, 0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX11-TRUE16-NEXT: s_min_i32 s4, s4, 13
+; GFX11-TRUE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX11-TRUE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s7
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s4, s6, s4
+; GFX11-TRUE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s0, 7
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s4, s5, s4
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX11-TRUE16-NEXT: s_cmpk_eq_i32 s2, 0x40f
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-LABEL: fptrunc_f64_to_f16_uniform:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_bfe_u32 s2, s1, 0xb0014
-; GFX1250-NEXT: s_lshr_b32 s3, s1, 8
-; GFX1250-NEXT: s_and_b32 s4, s1, 0x1ff
-; GFX1250-NEXT: s_addk_co_i32 s2, 0xfc10
-; GFX1250-NEXT: s_and_b32 s3, s3, 0xffe
-; GFX1250-NEXT: s_or_b32 s0, s4, s0
-; GFX1250-NEXT: s_cselect_b32 s0, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_or_b32 s0, s3, s0
-; GFX1250-NEXT: s_cselect_b32 s3, 1, 0
-; GFX1250-NEXT: s_sub_co_i32 s4, 1, s2
-; GFX1250-NEXT: s_or_b32 s5, s0, 0x1000
-; GFX1250-NEXT: s_max_i32 s4, s4, 0
-; GFX1250-NEXT: s_lshl_b32 s3, s3, 9
-; GFX1250-NEXT: s_min_i32 s4, s4, 13
-; GFX1250-NEXT: s_lshl_b32 s7, s2, 12
-; GFX1250-NEXT: s_lshr_b32 s6, s5, s4
-; GFX1250-NEXT: s_or_b32 s3, s3, 0x7c00
-; GFX1250-NEXT: s_lshl_b32 s4, s6, s4
-; GFX1250-NEXT: s_or_b32 s0, s0, s7
-; GFX1250-NEXT: s_cmp_lg_u32 s4, s5
-; GFX1250-NEXT: s_cselect_b32 s4, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_or_b32 s4, s6, s4
-; GFX1250-NEXT: s_cmp_lt_i32 s2, 1
-; GFX1250-NEXT: s_cselect_b32 s0, s4, s0
-; GFX1250-NEXT: s_and_b32 s4, s0, 7
-; GFX1250-NEXT: s_lshr_b32 s0, s0, 2
-; GFX1250-NEXT: s_cmp_eq_u32 s4, 3
-; GFX1250-NEXT: s_cselect_b32 s5, 1, 0
-; GFX1250-NEXT: s_cmp_gt_i32 s4, 5
-; GFX1250-NEXT: s_cselect_b32 s4, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_or_b32 s4, s5, s4
-; GFX1250-NEXT: s_cmp_lg_u32 s4, 0
-; GFX1250-NEXT: s_cselect_b32 s4, 1, 0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_add_co_i32 s0, s0, s4
-; GFX1250-NEXT: s_cmp_gt_i32 s2, 30
-; GFX1250-NEXT: s_cselect_b32 s0, 0x7c00, s0
-; GFX1250-NEXT: s_cmp_eq_u32 s2, 0x40f
-; GFX1250-NEXT: s_cselect_b32 s0, s3, s0
-; GFX1250-NEXT: s_lshr_b32 s1, s1, 16
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: s_and_b32 s1, s1, 0x8000
-; GFX1250-NEXT: s_or_b32 s0, s1, s0
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX12-FAKE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s2, 0xfc10
+; GFX12-FAKE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s4, s0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s3, s0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX12-FAKE16-NEXT: s_sub_co_i32 s4, 1, s2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX12-FAKE16-NEXT: s_max_i32 s4, s4, 0
+; GFX12-FAKE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX12-FAKE16-NEXT: s_min_i32 s4, s4, 13
+; GFX12-FAKE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX12-FAKE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX12-FAKE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s0, s7
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX12-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s4, s6, s4
+; GFX12-FAKE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s0, 7
+; GFX12-FAKE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-FAKE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX12-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX12-FAKE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX12-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_or_b32 s4, s5, s4
+; GFX12-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_add_co_i32 s0, s0, s4
+; GFX12-FAKE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX12-FAKE16-NEXT: s_cmp_eq_u32 s2, 0x40f
+; GFX12-FAKE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX12-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX12-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s2, 0xfc10
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s4, s0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s3, s0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX12-TRUE16-NEXT: s_sub_co_i32 s4, 1, s2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX12-TRUE16-NEXT: s_max_i32 s4, s4, 0
+; GFX12-TRUE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX12-TRUE16-NEXT: s_min_i32 s4, s4, 13
+; GFX12-TRUE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX12-TRUE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX12-TRUE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s0, s7
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX12-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s4, s6, s4
+; GFX12-TRUE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s0, 7
+; GFX12-TRUE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX12-TRUE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX12-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX12-TRUE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX12-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_or_b32 s4, s5, s4
+; GFX12-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX12-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_add_co_i32 s0, s0, s4
+; GFX12-TRUE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX12-TRUE16-NEXT: s_cmp_eq_u32 s2, 0x40f
+; GFX12-TRUE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX12-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-FAKE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX1250-FAKE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX1250-FAKE16-NEXT: s_addk_co_i32 s2, 0xfc10
+; GFX1250-FAKE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s4, s0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s3, s0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-FAKE16-NEXT: s_sub_co_i32 s4, 1, s2
+; GFX1250-FAKE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX1250-FAKE16-NEXT: s_max_i32 s4, s4, 0
+; GFX1250-FAKE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX1250-FAKE16-NEXT: s_min_i32 s4, s4, 13
+; GFX1250-FAKE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX1250-FAKE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX1250-FAKE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s0, s7
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_or_b32 s4, s6, s4
+; GFX1250-FAKE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX1250-FAKE16-NEXT: s_and_b32 s4, s0, 7
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX1250-FAKE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_or_b32 s4, s5, s4
+; GFX1250-FAKE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_add_co_i32 s0, s0, s4
+; GFX1250-FAKE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s2, 0x40f
+; GFX1250-FAKE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX1250-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX1250-FAKE16-NEXT: s_or_b32 s0, s1, s0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-TRUE16-LABEL: fptrunc_f64_to_f16_uniform:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: s_bfe_u32 s2, s1, 0xb0014
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX1250-TRUE16-NEXT: s_and_b32 s4, s1, 0x1ff
+; GFX1250-TRUE16-NEXT: s_addk_co_i32 s2, 0xfc10
+; GFX1250-TRUE16-NEXT: s_and_b32 s3, s3, 0xffe
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s4, s0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s3, s0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX1250-TRUE16-NEXT: s_sub_co_i32 s4, 1, s2
+; GFX1250-TRUE16-NEXT: s_or_b32 s5, s0, 0x1000
+; GFX1250-TRUE16-NEXT: s_max_i32 s4, s4, 0
+; GFX1250-TRUE16-NEXT: s_lshl_b32 s3, s3, 9
+; GFX1250-TRUE16-NEXT: s_min_i32 s4, s4, 13
+; GFX1250-TRUE16-NEXT: s_lshl_b32 s7, s2, 12
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s6, s5, s4
+; GFX1250-TRUE16-NEXT: s_or_b32 s3, s3, 0x7c00
+; GFX1250-TRUE16-NEXT: s_lshl_b32 s4, s6, s4
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s0, s7
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 s4, s5
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_or_b32 s4, s6, s4
+; GFX1250-TRUE16-NEXT: s_cmp_lt_i32 s2, 1
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s4, s0
+; GFX1250-TRUE16-NEXT: s_and_b32 s4, s0, 7
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s0, s0, 2
+; GFX1250-TRUE16-NEXT: s_cmp_eq_u32 s4, 3
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX1250-TRUE16-NEXT: s_cmp_gt_i32 s4, 5
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_or_b32 s4, s5, s4
+; GFX1250-TRUE16-NEXT: s_cmp_lg_u32 s4, 0
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_add_co_i32 s0, s0, s4
+; GFX1250-TRUE16-NEXT: s_cmp_gt_i32 s2, 30
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, 0x7c00, s0
+; GFX1250-TRUE16-NEXT: s_cmp_eq_u32 s2, 0x40f
+; GFX1250-TRUE16-NEXT: s_cselect_b32 s0, s3, s0
+; GFX1250-TRUE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: s_and_b32 s1, s1, 0x8000
+; GFX1250-TRUE16-NEXT: s_or_b32 s0, s1, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-TRUE16-NEXT: ; return to shader part epilog
%result = fptrunc double %a to half
ret half %result
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index bf5d5cb5462fe..360459a873a69 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -987,31 +987,29 @@ define i16 @v_fshl_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX11-TRUE16-LABEL: v_fshl_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_xor_b16 v2.h, v2.l, -1
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_xor_b16 v1.h, v4.l, -1
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v4.l, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v3.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, 9, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v2.h, v0.h, -1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 7
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, 1, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 1, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.l, 1, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v3.h, 7
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v2.h, 7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, v0.h, v1.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v2.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, v3.h, v3.l
; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 42c153ff5023a..6b5a188ab0378 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -985,32 +985,32 @@ define i16 @v_fshr_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
; GFX11-TRUE16-LABEL: v_fshr_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-TRUE16-NEXT: v_xor_b16 v1.h, v2.l, -1
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v3.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_xor_b16 v3.h, v2.l, -1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v3.l, -1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v4.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v3.l, 7
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
+; GFX11-TRUE16-NEXT: v_xor_b16 v2.h, v0.h, -1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 1, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 7
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, v0.h, v2.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v2.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v2.h, 7
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v3.h, 7
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, v0.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, v2.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, v3.h, v0.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
index b5ac435b16940..e1d82f640ea85 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fsub.ll
@@ -15,12 +15,19 @@ define amdgpu_ps half @fsub_s16_uniform(half inreg %a, half inreg %b) {
; GFX11-TRUE16-NEXT: v_sub_f16_e64 v0.l, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-LABEL: fsub_s16_uniform:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_sub_f16 s0, s0, s1
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: ; return to shader part epilog
+; GFX12-FAKE16-LABEL: fsub_s16_uniform:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_sub_f16 s0, s0, s1
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX12-TRUE16-LABEL: fsub_s16_uniform:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_sub_f16 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: ; return to shader part epilog
%fsub = fsub half %a, %b
ret half %fsub
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
index 0f963ba85b73e..445fb8061c3a2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
@@ -255,9 +255,9 @@ define amdgpu_ps void @insertelement_s_v2i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
; GFX11-NEXT: s_and_b32 s1, s4, 1
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v0.l
; GFX11-NEXT: s_lshl_b32 s1, s1, 4
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_lshl_b32 s2, 0xffff, s1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_not1_b32 s0, s0, s2
@@ -431,7 +431,7 @@ define amdgpu_ps void @insertelement_s_v2i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
; GFX11-NEXT: v_and_b32_e32 v1, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 4, v1
; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
@@ -612,11 +612,11 @@ define amdgpu_ps void @insertelement_v_v2i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v2i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v2.l
; GFX11-NEXT: s_and_b32 s0, s2, 1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v2, s0, v0
; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
@@ -702,7 +702,7 @@ define amdgpu_ps void @insertelement_v_v2i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v4, v[0:1], off
; GFX11-NEXT: v_and_b32_e32 v0, 1, v3
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v1, v2.l
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
@@ -982,7 +982,7 @@ define amdgpu_ps void @insertelement_s_v4i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11-NEXT: s_and_b32 s2, s4, 1
; GFX11-NEXT: s_lshr_b32 m0, s4, 1
; GFX11-NEXT: s_lshl_b32 s2, s2, 4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-NEXT: s_lshl_b32 s3, 0xffff, s2
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -1233,7 +1233,7 @@ define amdgpu_ps void @insertelement_s_v4i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
; GFX11-NEXT: v_and_b32_e32 v2, 1, v1
; GFX11-NEXT: v_lshrrev_b32_e32 v5, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v3, v0.l
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -1248,8 +1248,8 @@ define amdgpu_ps void @insertelement_s_v4i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(4) %ptr
@@ -1466,7 +1466,7 @@ define amdgpu_ps void @insertelement_v_v4i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v4i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX11-NEXT: s_and_b32 s0, s2, 1
; GFX11-NEXT: s_lshr_b32 m0, s2, 1
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
@@ -1585,21 +1585,20 @@ define amdgpu_ps void @insertelement_v_v4i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-NEXT: v_and_b32_e32 v4, 1, v3
; GFX11-NEXT: v_lshrrev_b32_e32 v6, 1, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_lshlrev_b32_e32 v4, 4, v4
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v3, v5
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
@@ -1943,15 +1942,14 @@ define amdgpu_ps void @insertelement_s_v8i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX11-NEXT: s_and_b32 s5, s4, 1
; GFX11-NEXT: s_lshr_b32 m0, s4, 1
; GFX11-NEXT: s_lshl_b32 s4, s5, 4
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: s_movrels_b32 s6, s0
-; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_and_not1_b32 s5, s6, s5
-; GFX11-NEXT: v_mov_b32_e32 v3, s3
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
; GFX11-NEXT: v_lshl_or_b32 v6, v0, s4, s5
; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2265,29 +2263,29 @@ define amdgpu_ps void @insertelement_s_v8i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX11-LABEL: insertelement_s_v8i16_v_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: v_lshrrev_b32_e32 v7, 1, v1
; GFX11-NEXT: v_and_b32_e32 v5, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_lshrrev_b32_e32 v7, 1, v1
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v4, s3
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 4, v5
+; GFX11-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_lshlrev_b32 v5, 4, v5
+; GFX11-NEXT: v_mov_b32_e32 v1, s0
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
-; GFX11-NEXT: v_lshlrev_b32_e64 v8, v5, 0xffff
; GFX11-NEXT: v_lshlrev_b32_e32 v0, v5, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e64 v8, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_not_b32_e32 v5, v8
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_or_b32 v8, v6, v5, v0
; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
; GFX11-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
@@ -2540,7 +2538,7 @@ define amdgpu_ps void @insertelement_v_v8i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX11-LABEL: insertelement_v_v8i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v2.l
; GFX11-NEXT: s_and_b32 s0, s2, 1
; GFX11-NEXT: s_lshr_b32 m0, s2, 1
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
@@ -2682,12 +2680,12 @@ define amdgpu_ps void @insertelement_v_v8i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v3
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v2.l
; GFX11-NEXT: v_mov_b32_e32 v9, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
; GFX11-NEXT: v_and_b32_e32 v0, 1, v3
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3129,20 +3127,20 @@ define amdgpu_ps void @insertelement_s_v16i16_v_s(ptr addrspace(4) inreg %ptr, i
; GFX11-NEXT: s_and_b32 s0, s4, 1
; GFX11-NEXT: s_lshr_b32 m0, s4, 1
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v8, v0.l
; GFX11-NEXT: s_lshl_b32 s1, 0xffff, s0
-; GFX11-NEXT: v_dual_mov_b32 v10, 16 :: v_dual_mov_b32 v9, 0
-; GFX11-NEXT: v_mov_b32_e32 v11, 0
+; GFX11-NEXT: v_dual_mov_b32 v10, 16 :: v_dual_mov_b32 v11, 0
+; GFX11-NEXT: v_mov_b32_e32 v9, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_movrels_b32 s2, s8
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s11
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX11-NEXT: s_and_not1_b32 s1, s2, s1
-; GFX11-NEXT: v_mov_b32_e32 v1, s9
+; GFX11-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
; GFX11-NEXT: v_lshl_or_b32 v12, v8, s0, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v5, s13
-; GFX11-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v7, s15
-; GFX11-NEXT: v_mov_b32_e32 v6, s14
+; GFX11-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v5, s13
+; GFX11-NEXT: v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s15
; GFX11-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_movreld_b32_e32 v0, v12
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b128 v[8:9], v[0:3], off
@@ -3627,22 +3625,20 @@ define amdgpu_ps void @insertelement_s_v16i16_v_v(ptr addrspace(4) inreg %ptr, i
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b256 s[0:7], s[2:3], 0x0
; GFX11-NEXT: v_lshrrev_b32_e32 v14, 1, v1
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-NEXT: v_dual_mov_b32 v12, 16 :: v_dual_mov_b32 v13, 0
; GFX11-NEXT: v_and_b32_e32 v1, 1, v1
-; GFX11-NEXT: v_mov_b32_e32 v13, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e64 v11, v1, 0xffff
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_not_b32_e32 v1, v11
-; GFX11-NEXT: v_mov_b32_e32 v11, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
-; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
-; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_lshlrev_b32 v0, v1, v0
+; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v9, s7
+; GFX11-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v7, s5
+; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v5, s3
+; GFX11-NEXT: v_mov_b32_e32 v4, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_cndmask_b32_e32 v10, v2, v3, vcc_lo
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v14
@@ -3653,19 +3649,20 @@ define amdgpu_ps void @insertelement_s_v16i16_v_v(ptr addrspace(4) inreg %ptr, i
; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v4, s0
; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v14
; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v14
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v1, v11
+; GFX11-NEXT: v_mov_b32_e32 v11, 0
; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v5, s1
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v6, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v6, s2
; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v7, s3
-; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v8, s4
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v8, s4
; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v9, s5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_and_or_b32 v15, v10, v1, v0
; GFX11-NEXT: v_mov_b32_e32 v10, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v12, 16 :: v_dual_cndmask_b32 v1, v3, v15
; GFX11-NEXT: v_cndmask_b32_e64 v0, v2, v15, s6
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v15, vcc_lo
; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, v15, s0
; GFX11-NEXT: v_cndmask_b32_e64 v3, v5, v15, s1
; GFX11-NEXT: v_cndmask_b32_e64 v4, v6, v15, s2
@@ -4031,14 +4028,14 @@ define amdgpu_ps void @insertelement_v_v16i16_v_s(ptr addrspace(1) %ptr, i16 %va
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
; GFX11-NEXT: global_load_b128 v[7:10], v[0:1], off offset:16
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v0, v2.l
; GFX11-NEXT: s_and_b32 s0, s2, 1
; GFX11-NEXT: s_lshr_b32 m0, s2, 1
; GFX11-NEXT: s_lshl_b32 s0, s0, 4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v11, 16 :: v_dual_lshlrev_b32 v0, s0, v0
-; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX11-NEXT: v_mov_b32_e32 v12, 0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s0, v0
+; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX11-NEXT: v_mov_b32_e32 v11, 16
; GFX11-NEXT: s_not_b32 s0, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_movrels_b32_e32 v1, v3
@@ -4248,36 +4245,36 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
; GFX11-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v3
; GFX11-NEXT: v_and_b32_e32 v3, 1, v3
-; GFX11-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v14, 16
-; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX11-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v14, 16
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 4, v3
; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 4, v0
; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 4, v3
; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 6, v0
+; GFX11-NEXT: v_lshlrev_b32_e64 v12, v3, 0xffff
; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v0
; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_lshlrev_b32 v2, v3, v2
-; GFX11-NEXT: v_lshlrev_b32_e64 v12, v3, 0xffff
+; GFX11-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_lshlrev_b32 v2, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_not_b32_e32 v3, v12
; GFX11-NEXT: s_waitcnt vmcnt(1)
; GFX11-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_cndmask_b32 v1, v4, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v6, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v7, s1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v8, s2
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s3
; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v10, s4
-; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v11, s5
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v11, s5
; GFX11-NEXT: v_and_or_b32 v16, v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_cndmask_b32_e64 v0, v4, v16, s6
; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v16, vcc_lo
; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, v16, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
index ddf02e41e6ce8..c4d316ca1a71c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/intrinsic-trunc.ll
@@ -62,7 +62,7 @@ define half @intrinsic_trunc_s16_s(half inreg %val) {
; GFX12-NEXT: s_trunc_f16 s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.trunc.f16(half %val)
ret half %result
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
index 1e3cbb574470b..3d83f6d4413be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.atomic.dim.a16.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10NSA %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX10NSA %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX11NSA %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=legalizer -o - %s | FileCheck -check-prefix=GFX12 %s
define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
@@ -51,9 +51,31 @@ define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_swap_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.swap.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_swap_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -65,10 +87,9 @@ define amdgpu_ps float @atomic_swap_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.swap.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -126,9 +147,31 @@ define amdgpu_ps float @atomic_add_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -140,10 +183,9 @@ define amdgpu_ps float @atomic_add_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -201,9 +243,31 @@ define amdgpu_ps float @atomic_sub_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_sub_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.sub.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_sub_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -215,10 +279,9 @@ define amdgpu_ps float @atomic_sub_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.sub.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -276,9 +339,31 @@ define amdgpu_ps float @atomic_smin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_smin_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smin.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_smin_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -290,10 +375,9 @@ define amdgpu_ps float @atomic_smin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smin.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -352,9 +436,31 @@ define amdgpu_ps float @atomic_umin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_umin_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umin.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_umin_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -366,10 +472,9 @@ define amdgpu_ps float @atomic_umin_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umin.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -427,9 +532,31 @@ define amdgpu_ps float @atomic_smax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_smax_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smax.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_smax_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -441,10 +568,9 @@ define amdgpu_ps float @atomic_smax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.smax.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -502,9 +628,31 @@ define amdgpu_ps float @atomic_umax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_umax_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umax.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_umax_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -516,10 +664,9 @@ define amdgpu_ps float @atomic_umax_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.umax.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -577,9 +724,31 @@ define amdgpu_ps float @atomic_and_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_and_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.and.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_and_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -591,10 +760,9 @@ define amdgpu_ps float @atomic_and_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.and.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -652,9 +820,31 @@ define amdgpu_ps float @atomic_or_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_or_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.or.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_or_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -666,10 +856,9 @@ define amdgpu_ps float @atomic_or_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s) {
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.or.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -727,9 +916,31 @@ define amdgpu_ps float @atomic_xor_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_xor_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.xor.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_xor_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -741,10 +952,9 @@ define amdgpu_ps float @atomic_xor_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.xor.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -802,9 +1012,31 @@ define amdgpu_ps float @atomic_inc_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_inc_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.inc.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_inc_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -816,10 +1048,9 @@ define amdgpu_ps float @atomic_inc_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.inc.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -877,9 +1108,31 @@ define amdgpu_ps float @atomic_dec_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_dec_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.dec.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_dec_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -891,10 +1144,9 @@ define amdgpu_ps float @atomic_dec_1d(<8 x i32> inreg %rsrc, i32 %data, i16 %s)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.dec.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -956,9 +1208,33 @@ define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_cmpswap_1d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_cmpswap_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -971,11 +1247,10 @@ define amdgpu_ps float @atomic_cmpswap_1d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.1d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1035,9 +1310,31 @@ define amdgpu_ps float @atomic_add_2d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_2d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1049,11 +1346,9 @@ define amdgpu_ps float @atomic_add_2d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1122,9 +1417,34 @@ define amdgpu_ps float @atomic_add_3d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_3d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.3d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1136,15 +1456,12 @@ define amdgpu_ps float @atomic_add_3d(<8 x i32> inreg %rsrc, i32 %data, i16 %s,
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.3d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1213,9 +1530,34 @@ define amdgpu_ps float @atomic_add_cube(<8 x i32> inreg %rsrc, i32 %data, i16 %s
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_cube
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.cube), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_cube
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1227,15 +1569,12 @@ define amdgpu_ps float @atomic_add_cube(<8 x i32> inreg %rsrc, i32 %data, i16 %s
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.cube), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1295,9 +1634,31 @@ define amdgpu_ps float @atomic_add_1darray(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_1darray
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1darray), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_1darray
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1309,11 +1670,9 @@ define amdgpu_ps float @atomic_add_1darray(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1darray), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1382,9 +1741,34 @@ define amdgpu_ps float @atomic_add_2darray(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_2darray
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darray), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_2darray
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1396,15 +1780,12 @@ define amdgpu_ps float @atomic_add_2darray(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darray), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1473,9 +1854,34 @@ define amdgpu_ps float @atomic_add_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_2dmsaa
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2dmsaa), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_2dmsaa
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1487,15 +1893,12 @@ define amdgpu_ps float @atomic_add_2dmsaa(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2dmsaa), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1566,9 +1969,34 @@ define amdgpu_ps float @atomic_add_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_2darraymsaa
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[COPY12]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darraymsaa), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_2darraymsaa
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1580,16 +2008,12 @@ define amdgpu_ps float @atomic_add_2darraymsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[COPY10]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY11]](i16), [[COPY12]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.2darraymsaa), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1647,9 +2071,31 @@ define amdgpu_ps float @atomic_add_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_add_1d_slc
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_add_1d_slc
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1661,10 +2107,9 @@ define amdgpu_ps float @atomic_add_1d_slc(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY9]](i32)
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY9]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.add.1d), [[COPY8]](i32), [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1728,9 +2173,33 @@ define amdgpu_ps float @atomic_cmpswap_2d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_cmpswap_2d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_cmpswap_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1743,12 +2212,10 @@ define amdgpu_ps float @atomic_cmpswap_2d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1821,9 +2288,36 @@ define amdgpu_ps float @atomic_cmpswap_3d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_cmpswap_3d
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
+ ; GFX11NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY12]](i16), [[DEF]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_cmpswap_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1836,16 +2330,13 @@ define amdgpu_ps float @atomic_cmpswap_3d(<8 x i32> inreg %rsrc, i32 %cmp, i32 %
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY12]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.3d), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
@@ -1920,9 +2411,36 @@ define amdgpu_ps float @atomic_cmpswap_2darraymsaa(<8 x i32> inreg %rsrc, i32 %c
; GFX10NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
; GFX10NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
+ ; GFX11NSA-LABEL: name: atomic_cmpswap_2darraymsaa
+ ; GFX11NSA: bb.1.main_body:
+ ; GFX11NSA-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
+ ; GFX11NSA-NEXT: {{ $}}
+ ; GFX11NSA-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
+ ; GFX11NSA-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
+ ; GFX11NSA-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY $sgpr4
+ ; GFX11NSA-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY $sgpr5
+ ; GFX11NSA-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY $sgpr6
+ ; GFX11NSA-NEXT: [[COPY5:%[0-9]+]]:_(i32) = COPY $sgpr7
+ ; GFX11NSA-NEXT: [[COPY6:%[0-9]+]]:_(i32) = COPY $sgpr8
+ ; GFX11NSA-NEXT: [[COPY7:%[0-9]+]]:_(i32) = COPY $sgpr9
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
+ ; GFX11NSA-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
+ ; GFX11NSA-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
+ ; GFX11NSA-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX11NSA-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX11NSA-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX11NSA-NEXT: [[COPY13:%[0-9]+]]:_(i16) = COPY $vgpr5_lo16
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
+ ; GFX11NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY12]](i16), [[COPY13]](i16)
+ ; GFX11NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
+ ; GFX11NSA-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
+ ; GFX11NSA-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
+ ; GFX11NSA-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ;
; GFX12-LABEL: name: atomic_cmpswap_2darraymsaa
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1935,17 +2453,13 @@ define amdgpu_ps float @atomic_cmpswap_2darraymsaa(<8 x i32> inreg %rsrc, i32 %c
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i16) = COPY $vgpr5_lo16
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY10]](i16), [[COPY11]](i16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[COPY12]](i16), [[COPY13]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.atomic.cmpswap.2darraymsaa), [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (volatile dereferenceable load store (i32), addrspace 8)
; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f32) = G_BITCAST [[AMDGPU_INTRIN_IMAGE_LOAD]](i32)
; GFX12-NEXT: $vgpr0 = COPY [[BITCAST]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
index 61006f8c1eab6..b55233b3cfc30 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.a16.ll
@@ -67,7 +67,7 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX11-LABEL: name: sample_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -83,11 +83,9 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -98,7 +96,7 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX12-LABEL: name: sample_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -114,11 +112,9 @@ define amdgpu_ps <4 x float> @sample_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -196,7 +192,7 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX11-LABEL: name: sample_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -212,13 +208,9 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -229,7 +221,7 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX12-LABEL: name: sample_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -245,13 +237,9 @@ define amdgpu_ps <4 x float> @sample_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -338,7 +326,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX11-LABEL: name: sample_3d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -354,18 +342,12 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -376,7 +358,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
;
; GFX12-LABEL: name: sample_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -392,18 +374,12 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -490,7 +466,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_cube
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -506,18 +482,12 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -528,7 +498,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_cube
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -544,18 +514,12 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cube), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -633,7 +597,7 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_1darray
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -649,13 +613,9 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -666,7 +626,7 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_1darray
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -682,13 +642,9 @@ define amdgpu_ps <4 x float> @sample_1darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.1darray), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -775,7 +731,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_2darray
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -791,18 +747,12 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -813,7 +763,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_2darray
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -829,18 +779,12 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.2darray), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -921,7 +865,7 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_c_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -938,13 +882,11 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -954,7 +896,7 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_c_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -971,13 +913,11 @@ define amdgpu_ps <4 x float> @sample_c_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1059,7 +999,7 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_c_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1076,15 +1016,11 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1094,7 +1030,7 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_c_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1111,15 +1047,11 @@ define amdgpu_ps <4 x float> @sample_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1196,7 +1128,7 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1212,13 +1144,9 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1229,7 +1157,7 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1245,13 +1173,9 @@ define amdgpu_ps <4 x float> @sample_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1338,7 +1262,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1354,18 +1278,12 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1376,7 +1294,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1392,18 +1310,12 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1486,7 +1398,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1503,15 +1415,11 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1521,7 +1429,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1538,15 +1446,11 @@ define amdgpu_ps <4 x float> @sample_c_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1636,7 +1540,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1653,20 +1557,14 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1676,7 +1574,7 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1693,20 +1591,14 @@ define amdgpu_ps <4 x float> @sample_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1788,7 +1680,7 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_b_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1804,15 +1696,11 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1823,7 +1711,7 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_b_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1839,15 +1727,11 @@ define amdgpu_ps <4 x float> @sample_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1934,7 +1818,7 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_b_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1950,18 +1834,12 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -1972,7 +1850,7 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_b_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1988,18 +1866,12 @@ define amdgpu_ps <4 x float> @sample_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2086,7 +1958,7 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_b_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2102,18 +1974,14 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2123,7 +1991,7 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_b_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2139,18 +2007,14 @@ define amdgpu_ps <4 x float> @sample_c_b_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2240,7 +2104,7 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_b_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2256,21 +2120,15 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2280,7 +2138,7 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_b_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2296,21 +2154,15 @@ define amdgpu_ps <4 x float> @sample_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2396,7 +2248,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_b_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2412,18 +2264,12 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2434,7 +2280,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_b_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2450,18 +2296,12 @@ define amdgpu_ps <4 x float> @sample_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2554,7 +2394,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_b_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2570,22 +2410,14 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2596,7 +2428,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_b_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2612,22 +2444,14 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -2718,7 +2542,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_b_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2734,21 +2558,15 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2758,7 +2576,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_b_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2774,21 +2592,15 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2884,7 +2696,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_b_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2900,25 +2712,17 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2928,7 +2732,7 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_b_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr1, $vgpr0_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2944,25 +2748,17 @@ define amdgpu_ps <4 x float> @sample_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[BITCAST2]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.b.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3050,7 +2846,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3066,19 +2862,13 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3089,7 +2879,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3105,19 +2895,13 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3216,7 +3000,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3232,27 +3016,15 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3263,7 +3035,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3279,27 +3051,15 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -3419,7 +3179,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_3d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3435,40 +3195,22 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
- ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX11-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -3480,7 +3222,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3496,40 +3238,22 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
- ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX12-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR5]](<2 x f16>), [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -3623,7 +3347,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_d_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3640,21 +3364,15 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3664,7 +3382,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_d_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3681,21 +3399,15 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3797,7 +3509,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_d_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3814,29 +3526,17 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3846,7 +3546,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_d_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3863,29 +3563,17 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -3977,7 +3665,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_d_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -3993,22 +3681,14 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4019,7 +3699,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_d_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4035,22 +3715,14 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4157,7 +3829,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_d_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4173,32 +3845,18 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4209,7 +3867,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_d_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4225,32 +3883,18 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4347,7 +3991,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4364,24 +4008,16 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4391,7 +4027,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_d_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4408,24 +4044,16 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4535,7 +4163,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4552,34 +4180,20 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY19]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4589,7 +4203,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_d_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4606,35 +4220,21 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY19]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -4722,7 +4322,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cd_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4738,19 +4338,13 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4761,7 +4355,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cd_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4777,19 +4371,13 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4888,7 +4476,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cd_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4904,27 +4492,15 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -4935,7 +4511,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cd_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -4951,27 +4527,15 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5064,7 +4628,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cd_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5081,21 +4645,15 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5105,7 +4663,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cd_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5122,21 +4680,15 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5238,7 +4790,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cd_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5255,29 +4807,17 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5287,7 +4827,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cd_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5304,29 +4844,17 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST6]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5418,7 +4946,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-LABEL: name: sample_cd_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5434,22 +4962,14 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5460,7 +4980,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-LABEL: name: sample_cd_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5476,22 +4996,14 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5598,7 +5110,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-LABEL: name: sample_cd_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5614,32 +5126,18 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5650,7 +5148,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-LABEL: name: sample_cd_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5666,32 +5164,18 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -5788,7 +5272,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_cd_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5805,24 +5289,16 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5832,7 +5308,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_cd_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5849,24 +5325,16 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -5976,7 +5444,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_cd_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -5993,34 +5461,20 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY19]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6030,7 +5484,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_cd_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6047,35 +5501,21 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[COPY18]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY19]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST7]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6152,7 +5592,7 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_l_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6168,13 +5608,9 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6185,7 +5621,7 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_l_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6201,13 +5637,9 @@ define amdgpu_ps <4 x float> @sample_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6294,7 +5726,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_l_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6310,18 +5742,12 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6332,7 +5758,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_l_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6348,18 +5774,12 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.l.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6442,7 +5862,7 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_l_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6459,15 +5879,11 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6477,7 +5893,7 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_l_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6494,15 +5910,11 @@ define amdgpu_ps <4 x float> @sample_c_l_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6592,7 +6004,7 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_l_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6609,20 +6021,14 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6632,7 +6038,7 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_l_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6649,20 +6055,14 @@ define amdgpu_ps <4 x float> @sample_c_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.l.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -6737,7 +6137,7 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_lz_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6753,11 +6153,9 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6768,7 +6166,7 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_lz_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6784,11 +6182,9 @@ define amdgpu_ps <4 x float> @sample_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6866,7 +6262,7 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_lz_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6882,13 +6278,9 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -6899,7 +6291,7 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_lz_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -6915,13 +6307,9 @@ define amdgpu_ps <4 x float> @sample_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.lz.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -7002,7 +6390,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_lz_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7019,13 +6407,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7035,7 +6421,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_lz_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7052,13 +6438,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7140,7 +6524,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_lz_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7157,15 +6541,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7175,7 +6555,7 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_lz_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7192,15 +6572,11 @@ define amdgpu_ps <4 x float> @sample_c_lz_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.lz.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7307,7 +6683,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_o_2darray_V1
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7325,42 +6701,28 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
; GFX11-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX12-LABEL: name: sample_c_d_o_2darray_V1
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7378,36 +6740,22 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
main_body:
@@ -7514,7 +6862,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX11-LABEL: name: sample_c_d_o_2darray_V2
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7532,36 +6880,22 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -7569,7 +6903,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12-LABEL: name: sample_c_d_o_2darray_V2
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -7587,36 +6921,22 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
index 4934f93ac18df..6ee08839fbb93 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.a16.ll
@@ -42,7 +42,7 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX11-LABEL: name: sample_d_1d_g16_a16
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -58,19 +58,13 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -81,7 +75,7 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX12-LABEL: name: sample_d_1d_g16_a16
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -97,19 +91,13 @@ define amdgpu_ps <4 x float> @sample_d_1d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -166,7 +154,7 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX11-LABEL: name: sample_d_2d_g16_a16
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -182,27 +170,15 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -213,7 +189,7 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX12-LABEL: name: sample_d_2d_g16_a16
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -229,27 +205,15 @@ define amdgpu_ps <4 x float> @sample_d_2d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST4]](f16), [[BITCAST5]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -317,7 +281,7 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX11-LABEL: name: sample_d_3d_g16_a16
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -333,40 +297,22 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX11-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX11-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
- ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX11-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -378,7 +324,7 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
;
; GFX12-LABEL: name: sample_d_3d_g16_a16
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16, $vgpr6_lo16, $vgpr7_lo16, $vgpr8_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -394,40 +340,22 @@ define amdgpu_ps <4 x float> @sample_d_3d_g16_a16(<8 x i32> inreg %rsrc, <4 x i3
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
- ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(i32) = COPY $vgpr6
- ; GFX12-NEXT: [[TRUNC6:%[0-9]+]]:_(i16) = G_TRUNC [[COPY18]](i32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC6]](i16)
- ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(i32) = COPY $vgpr7
- ; GFX12-NEXT: [[TRUNC7:%[0-9]+]]:_(i16) = G_TRUNC [[COPY19]](i32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC7]](i16)
- ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(i32) = COPY $vgpr8
- ; GFX12-NEXT: [[TRUNC8:%[0-9]+]]:_(i16) = G_TRUNC [[COPY20]](i32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC8]](i16)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
+ ; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f16) = COPY $vgpr6_lo16
+ ; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f16) = COPY $vgpr7_lo16
+ ; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f16) = COPY $vgpr8_lo16
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST6]](f16), [[BITCAST7]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST8]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR6:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY18]](f16), [[COPY19]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR7:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY20]](f16), [[DEF]](f16)
; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR5]](<2 x f16>), [[BUILD_VECTOR6]](<2 x f16>), [[BUILD_VECTOR7]](<2 x f16>)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
index 2ab7e0d348f41..12df216b7d7e2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.sample.g16.ll
@@ -41,7 +41,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -57,18 +57,14 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -78,7 +74,7 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -94,18 +90,14 @@ define amdgpu_ps <4 x float> @sample_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -160,7 +152,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -176,25 +168,17 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -204,7 +188,7 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -220,25 +204,17 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -303,7 +279,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-LABEL: name: sample_d_3d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr6, $vgpr7, $vgpr8, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -319,36 +295,24 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[BUILD_VECTOR5]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -359,7 +323,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-LABEL: name: sample_d_3d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr6, $vgpr7, $vgpr8, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -375,36 +339,24 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC4:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC4]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC5:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC5]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST3]](f16), [[BITCAST4]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST5]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR4:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR5:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY17]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR5]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.3d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BUILD_VECTOR4]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
@@ -457,7 +409,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_d_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -474,19 +426,15 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -496,7 +444,7 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_d_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -513,19 +461,15 @@ define amdgpu_ps <4 x float> @sample_c_d_1d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -582,7 +526,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-LABEL: name: sample_c_d_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -599,26 +543,18 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -628,7 +564,7 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-LABEL: name: sample_c_d_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -645,27 +581,19 @@ define amdgpu_ps <4 x float> @sample_c_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -717,7 +645,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_d_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr3, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -733,20 +661,16 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -756,7 +680,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_d_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr3, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -772,20 +696,16 @@ define amdgpu_ps <4 x float> @sample_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -842,7 +762,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_d_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -858,27 +778,19 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -888,7 +800,7 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_d_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -904,28 +816,20 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.d.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -979,7 +883,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr4, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -996,21 +900,17 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1020,7 +920,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_d_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr4, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1037,22 +937,18 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1112,7 +1008,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr7, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1129,29 +1025,21 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1161,7 +1049,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_d_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr7, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1178,29 +1066,21 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1250,7 +1130,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cd_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1266,18 +1146,14 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1287,7 +1163,7 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cd_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1303,18 +1179,14 @@ define amdgpu_ps <4 x float> @sample_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1369,7 +1241,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-LABEL: name: sample_cd_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1385,25 +1257,17 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1413,7 +1277,7 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-LABEL: name: sample_cd_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1429,25 +1293,17 @@ define amdgpu_ps <4 x float> @sample_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1499,7 +1355,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cd_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1516,19 +1372,15 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1538,7 +1390,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cd_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1555,19 +1407,15 @@ define amdgpu_ps <4 x float> @sample_c_cd_1d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1624,7 +1472,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-LABEL: name: sample_c_cd_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1641,26 +1489,18 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1670,7 +1510,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-LABEL: name: sample_c_cd_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1687,27 +1527,19 @@ define amdgpu_ps <4 x float> @sample_c_cd_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1759,7 +1591,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-LABEL: name: sample_cd_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr3, $vgpr0_lo16, $vgpr1_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1775,20 +1607,16 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1798,7 +1626,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-LABEL: name: sample_cd_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr2, $vgpr3, $vgpr0_lo16, $vgpr1_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1814,20 +1642,16 @@ define amdgpu_ps <4 x float> @sample_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f32) = COPY $vgpr2
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY14]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.1d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1884,7 +1708,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-LABEL: name: sample_cd_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1900,27 +1724,19 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -1930,7 +1746,7 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-LABEL: name: sample_cd_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -1946,28 +1762,20 @@ define amdgpu_ps <4 x float> @sample_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $sgpr12
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f16) = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY12]](f16), [[COPY13]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.cd.cl.2d), 15, [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2021,7 +1829,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_cd_cl_1d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr4, $vgpr1_lo16, $vgpr2_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2038,21 +1846,17 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX11-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2062,7 +1866,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_cd_cl_1d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr3, $vgpr4, $vgpr1_lo16, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2079,22 +1883,18 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_1d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f32) = COPY $vgpr3
; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f32) = COPY $vgpr4
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(f16) = G_IMPLICIT_DEF
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST1]](f16), [[DEF]](f16)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST2]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[DEF]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY15]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY16]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.1d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2154,7 +1954,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_cd_cl_2d
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr7, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2171,29 +1971,21 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[CONCAT_VECTORS]](<4 x f16>), $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2203,7 +1995,7 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-LABEL: name: sample_c_cd_cl_2d
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr5, $vgpr6, $vgpr7, $vgpr1_lo16, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2220,29 +2012,21 @@ define amdgpu_ps <4 x float> @sample_c_cd_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[COPY11:%[0-9]+]]:_(i32) = COPY $sgpr13
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(f32) = COPY $vgpr0
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY13]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f16) = COPY $vgpr1_lo16
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f32) = COPY $vgpr5
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST4]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY13]](f16), [[COPY14]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY15]](f16), [[COPY16]](f16)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY17]](f32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.cd.cl.2d), 15, [[BITCAST]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2300,7 +2084,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-LABEL: name: sample_c_d_o_2darray_V1
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr6, $vgpr7, $vgpr8, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2318,36 +2102,28 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
; GFX11-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX11-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX12-LABEL: name: sample_c_d_o_2darray_V1
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr6, $vgpr7, $vgpr8, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2365,30 +2141,22 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 4, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (f32), addrspace 8)
; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_INTRIN_IMAGE_LOAD]](f32)
; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
main_body:
@@ -2444,7 +2212,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX11-LABEL: name: sample_c_d_o_2darray_V2
; GFX11: bb.1.main_body:
- ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX11-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr6, $vgpr7, $vgpr8, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX11-NEXT: {{ $}}
; GFX11-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX11-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2462,30 +2230,22 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX11-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX11-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX11-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX11-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX11-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX11-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX11-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX11-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX11-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX11-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX11-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX11-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX11-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX11-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX11-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX11-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX11-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX11-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
- ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
+ ; GFX11-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX11-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX11-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX11-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX11-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX11-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX11-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX11-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x f16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
+ ; GFX11-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[BUILD_VECTOR3]](<2 x f16>), [[CONCAT_VECTORS]](<6 x f16>), $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX11-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX11-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX11-NEXT: $vgpr1 = COPY [[UV1]](f32)
@@ -2493,7 +2253,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12-LABEL: name: sample_c_d_o_2darray_V2
; GFX12: bb.1.main_body:
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $sgpr10, $sgpr11, $sgpr12, $sgpr13, $vgpr0, $vgpr1, $vgpr6, $vgpr7, $vgpr8, $vgpr2_lo16, $vgpr3_lo16, $vgpr4_lo16, $vgpr5_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -2511,30 +2271,22 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32), [[COPY10]](i32), [[COPY11]](i32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(f32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY14]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(i32) = COPY $vgpr3
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[COPY15]](i32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC1]](i16)
- ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(i32) = COPY $vgpr4
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[COPY16]](i32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC2]](i16)
- ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(i32) = COPY $vgpr5
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[COPY17]](i32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC3]](i16)
+ ; GFX12-NEXT: [[COPY14:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: [[COPY15:%[0-9]+]]:_(f16) = COPY $vgpr3_lo16
+ ; GFX12-NEXT: [[COPY16:%[0-9]+]]:_(f16) = COPY $vgpr4_lo16
+ ; GFX12-NEXT: [[COPY17:%[0-9]+]]:_(f16) = COPY $vgpr5_lo16
; GFX12-NEXT: [[COPY18:%[0-9]+]]:_(f32) = COPY $vgpr6
; GFX12-NEXT: [[COPY19:%[0-9]+]]:_(f32) = COPY $vgpr7
; GFX12-NEXT: [[COPY20:%[0-9]+]]:_(f32) = COPY $vgpr8
- ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
- ; GFX12-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
- ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST]](f16), [[BITCAST1]](f16)
- ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[BITCAST2]](f16), [[BITCAST3]](f16)
- ; GFX12-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
- ; GFX12-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
- ; GFX12-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
- ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST6]](<2 x f16>), [[BITCAST7]](<2 x f16>), [[BITCAST8]](<2 x f16>)
- ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST4]](<2 x f16>), [[BITCAST5]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY12]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY13]](f32)
+ ; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY14]](f16), [[COPY15]](f16)
+ ; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x f16>) = G_BUILD_VECTOR [[COPY16]](f16), [[COPY17]](f16)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY18]](f32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY19]](f32)
+ ; GFX12-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[COPY20]](f32)
+ ; GFX12-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR3]](<2 x f16>), [[BITCAST2]](<2 x f16>), [[BITCAST3]](<2 x f16>), [[BITCAST4]](<2 x f16>)
+ ; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.sample.c.d.o.2darray), 6, [[BITCAST]](<2 x f16>), [[BITCAST1]](<2 x f16>), [[BUILD_VECTOR2]](<2 x f16>), [[CONCAT_VECTORS]](<8 x f16>), $noreg, $noreg, $noreg, $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), [[BUILD_VECTOR1]](<4 x i32>), 0, 0, 0, 2 :: (dereferenceable load (<2 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<2 x f32>)
; GFX12-NEXT: $vgpr0 = COPY [[UV]](f32)
; GFX12-NEXT: $vgpr1 = COPY [[UV1]](f32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
index 9657f2ec27268..5cc6d9c6b9f7b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
@@ -72,7 +72,7 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
;
; GFX12-LABEL: name: image_store_f16
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7, $sgpr8, $sgpr9, $vgpr0, $vgpr1, $vgpr2_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $sgpr3
@@ -85,10 +85,8 @@ define amdgpu_ps void @image_store_f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t, ha
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(i32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY $vgpr2
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[COPY10]](i32)
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(f16) = G_BITCAST [[TRUNC]](i16)
- ; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[BITCAST]](f16), 1, [[COPY8]](i32), [[COPY9]](i32), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(f16) = COPY $vgpr2_lo16
+ ; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE_D16 intrinsic(@llvm.amdgcn.image.store.2d), [[COPY10]](f16), 1, [[COPY8]](i32), [[COPY9]](i32), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable store (f16), addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.image.store.2d.f16.i32(half %data, i32 1, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
index ac1b50a967ebe..177edf7dae535 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.fract.ll
@@ -13,7 +13,7 @@ define amdgpu_ps half @s_fract_f16(half inreg %src) {
; GFX12-NEXT: v_readfirstlane_b32 s0, v0
; GFX12-NEXT: s_add_f16 s0, s0, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: ; return to shader part epilog
%fract = call half @llvm.amdgcn.fract.f16(half %src)
%res = fadd half %fract, %fract
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
index 394d145f6064d..33513214e779b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.a16.ll
@@ -1420,6 +1420,7 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX11-TRUE16-LABEL: atomic_add_i32_3d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1428,7 +1429,7 @@ define amdgpu_ps float @atomic_add_i32_3d(<8 x i32> inreg %rsrc, i32 %data, i16
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_3D unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1528,6 +1529,7 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i1
; GFX11-TRUE16-LABEL: atomic_add_i32_cube:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1536,7 +1538,7 @@ define amdgpu_ps float @atomic_add_i32_cube(<8 x i32> inreg %rsrc, i32 %data, i1
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_CUBE unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1736,6 +1738,7 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX11-TRUE16-LABEL: atomic_add_i32_2darray:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1744,7 +1747,7 @@ define amdgpu_ps float @atomic_add_i32_2darray(<8 x i32> inreg %rsrc, i32 %data,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1844,6 +1847,7 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX11-TRUE16-LABEL: atomic_add_i32_2dmsaa:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1852,7 +1856,7 @@ define amdgpu_ps float @atomic_add_i32_2dmsaa(<8 x i32> inreg %rsrc, i32 %data,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, [v1, v3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1951,9 +1955,9 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
;
; GFX11-TRUE16-LABEL: atomic_add_i32_2darraymsaa:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v6, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1962,7 +1966,7 @@ define amdgpu_ps float @atomic_add_i32_2darraymsaa(<8 x i32> inreg %rsrc, i32 %d
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v0, v[5:6], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v0, v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -3535,6 +3539,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
; GFX11-TRUE16-LABEL: atomic_add_i64_3d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -3543,7 +3548,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_3d(<8 x i32> inreg %rsrc, i64 %data
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_3D unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -3643,6 +3648,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
; GFX11-TRUE16-LABEL: atomic_add_i64_cube:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -3651,7 +3657,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_cube(<8 x i32> inreg %rsrc, i64 %da
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_CUBE unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -3851,6 +3857,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
; GFX11-TRUE16-LABEL: atomic_add_i64_2darray:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -3859,7 +3866,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darray(<8 x i32> inreg %rsrc, i64
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_ARRAY unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -3959,6 +3966,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
; GFX11-TRUE16-LABEL: atomic_add_i64_2dmsaa:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -3967,7 +3975,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2dmsaa(<8 x i32> inreg %rsrc, i64 %
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], [v2, v4], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -4066,9 +4074,9 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
;
; GFX11-TRUE16-LABEL: atomic_add_i64_2darraymsaa:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v7, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -4077,7 +4085,7 @@ define amdgpu_ps <2 x float> @atomic_add_i64_2darraymsaa(<8 x i32> inreg %rsrc,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[6:7], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
+; GFX11-TRUE16-NEXT: image_atomic_add v[0:1], v[2:3], s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm glc a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
index 0cd3a5d235ea6..32eb64298cd40 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -55,9 +55,11 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: gather4_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -69,8 +71,6 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -100,9 +100,11 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12-TRUE16-LABEL: gather4_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -114,8 +116,6 @@ define amdgpu_ps <4 x float> @gather4_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4 v[0:3], v0, s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -200,9 +200,12 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-TRUE16-LABEL: gather4_cube:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -214,9 +217,7 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -247,9 +248,11 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-TRUE16-LABEL: gather4_cube:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -261,8 +264,6 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -349,9 +350,12 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-TRUE16-LABEL: gather4_2darray:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -363,9 +367,7 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -396,9 +398,11 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-TRUE16-LABEL: gather4_2darray:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -410,8 +414,6 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4 v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -494,9 +496,11 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-TRUE16-LABEL: gather4_c_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -508,8 +512,6 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_c v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -539,9 +541,11 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-TRUE16-LABEL: gather4_c_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -553,8 +557,6 @@ define amdgpu_ps <4 x float> @gather4_c_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_c v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -639,9 +641,12 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX11-TRUE16-LABEL: gather4_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -653,9 +658,7 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -686,9 +689,11 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
;
; GFX12-TRUE16-LABEL: gather4_cl_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -700,8 +705,6 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_cl v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -788,9 +791,11 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-TRUE16-LABEL: gather4_c_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -802,8 +807,6 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -835,9 +838,11 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-TRUE16-LABEL: gather4_c_cl_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -849,8 +854,6 @@ define amdgpu_ps <4 x float> @gather4_c_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_c_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -937,9 +940,11 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-TRUE16-LABEL: gather4_b_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -951,8 +956,6 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_b v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -984,9 +987,11 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX12-TRUE16-LABEL: gather4_b_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -998,8 +1003,6 @@ define amdgpu_ps <4 x float> @gather4_b_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_b v[0:3], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -1086,9 +1089,11 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-TRUE16-LABEL: gather4_c_b_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1100,8 +1105,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_c_b v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1133,9 +1136,11 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX12-TRUE16-LABEL: gather4_c_b_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1147,8 +1152,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_c_b v[0:3], [v0, v1, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -1237,9 +1240,12 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX11-TRUE16-LABEL: gather4_b_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1251,9 +1257,7 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], v[0:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1286,9 +1290,11 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
;
; GFX12-TRUE16-LABEL: gather4_b_cl_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1300,8 +1306,6 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_b_cl v[0:3], [v0, v1, v3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -1392,9 +1396,11 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-TRUE16-LABEL: gather4_c_b_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX11-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1406,8 +1412,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX11-TRUE16-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1441,9 +1445,11 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-TRUE16-LABEL: gather4_c_b_cl_2d:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: s_mov_b32 s14, exec_lo
+; GFX12-TRUE16-NEXT: s_mov_b32 s1, exec_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX12-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s1
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s2, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s3, s5
@@ -1455,8 +1461,6 @@ define amdgpu_ps <4 x float> @gather4_c_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX12-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
; GFX12-TRUE16-NEXT: image_gather4_c_b_cl v[0:3], [v0, v1, v2, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -1540,6 +1544,7 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-LABEL: gather4_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -1552,7 +1557,7 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s9, s11
; GFX11-TRUE16-NEXT: s_mov_b32 s10, s12
; GFX11-TRUE16-NEXT: s_mov_b32 s11, s13
-; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], [v0, v2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
index 7ca3ae1561b2f..b633a37dd22be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.2darraymsaa.a16.ll
@@ -45,10 +45,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
;
; GFX11-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -57,7 +56,7 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw(<8 x i32> inreg %rsrc,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_load v[0:3], v[4:5], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX11-TRUE16-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -179,9 +178,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
; GFX11-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
@@ -237,9 +236,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe(<8 x i32> inreg %r
; GFX12-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v7
@@ -361,9 +360,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
; GFX11-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
@@ -419,9 +418,9 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_v4f32_xyzw_tfe_lwe(<8 x i32> inre
; GFX12-TRUE16-LABEL: load_2darraymsaa_v4f32_xyzw_tfe_lwe:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.h, v3.l
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v7
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
index cd1b7227e427a..b337e5db23f5d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.3d.a16.ll
@@ -46,6 +46,7 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
; GFX11-TRUE16-LABEL: load_3d_v4f32_xyzw:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s1, s3
; GFX11-TRUE16-NEXT: s_mov_b32 s2, s4
@@ -54,7 +55,7 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw(<8 x i32> inreg %rsrc, i16 %s,
; GFX11-TRUE16-NEXT: s_mov_b32 s5, s7
; GFX11-TRUE16-NEXT: s_mov_b32 s6, s8
; GFX11-TRUE16-NEXT: s_mov_b32 s7, s9
-; GFX11-TRUE16-NEXT: image_load v[0:3], [v0, v2], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
+; GFX11-TRUE16-NEXT: image_load v[0:3], v[0:1], s[0:7] dmask:0xf dim:SQ_RSRC_IMG_3D unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -175,10 +176,10 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
; GFX11-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, v7
@@ -228,10 +229,10 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe(<8 x i32> inreg %rsrc, ptr
; GFX12-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v7 :: v_dual_mov_b32 v11, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
@@ -343,10 +344,10 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
; GFX11-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v8, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, v7
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, v7
@@ -396,10 +397,10 @@ define amdgpu_ps <4 x float> @load_3d_v4f32_xyzw_tfe_lwe(<8 x i32> inreg %rsrc,
; GFX12-TRUE16-LABEL: load_3d_v4f32_xyzw_tfe_lwe:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, s2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v7 :: v_dual_mov_b32 v9, v7
; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, v7 :: v_dual_mov_b32 v11, v7
; GFX12-TRUE16-NEXT: s_mov_b32 s1, s3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
index 096cb9ff625da..d8866497d2169 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.sample.g16.ll
@@ -496,11 +496,12 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v4
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
-; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[4:7]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
@@ -554,11 +555,12 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v4
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
-; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v7
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v8
+; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[4:7]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
index 225ae801fc12b..42469f8f78c47 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.interp.inreg.ll
@@ -397,7 +397,7 @@ define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #
; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
; GFX12-TRUE16-NEXT: s_add_f16 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-TRUE16-NEXT: ; return to shader part epilog
;
; GFX12-FAKE16-LABEL: v_interp_f16_imm_params:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
index 04cdd48f8f78e..faae4bbd9884b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.make.buffer.rsrc.ll
@@ -694,19 +694,21 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-LABEL: name: general_case_load_with_waterfall
; CHECK45: bb.1 (%ir-block.0):
; CHECK45-NEXT: successors: %bb.2(0x80000000)
- ; CHECK45-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; CHECK45-NEXT: liveins: $vgpr0, $vgpr1, $vgpr3, $vgpr4, $vgpr5, $vgpr2_lo16
; CHECK45-NEXT: {{ $}}
; CHECK45-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; CHECK45-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; CHECK45-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
- ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; CHECK45-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY $vgpr2_lo16
; CHECK45-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
; CHECK45-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; CHECK45-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; CHECK45-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY2]], %subreg.lo16, [[DEF]], %subreg.hi16
; CHECK45-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 8191
; CHECK45-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; CHECK45-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY4]], [[COPY6]], implicit $exec
- ; CHECK45-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[V_AND_B32_e64_]], %subreg.sub1
+ ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY3]], %subreg.sub0, [[V_AND_B32_e64_]], %subreg.sub1
; CHECK45-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; CHECK45-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 25
; CHECK45-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_2]]
@@ -715,17 +717,17 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-NEXT: [[V_LSHL_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHL_OR_B32_e64 [[COPY3]], [[COPY7]], [[COPY9]], implicit $exec
; CHECK45-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 7
; CHECK45-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_3]]
- ; CHECK45-NEXT: [[V_LSHRREV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_LSHRREV_B64_e64 [[COPY10]], [[REG_SEQUENCE1]], implicit $exec
+ ; CHECK45-NEXT: [[V_LSHRREV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_LSHRREV_B64_e64 [[COPY10]], [[REG_SEQUENCE2]], implicit $exec
; CHECK45-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 12
; CHECK45-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_4]]
- ; CHECK45-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY11]], [[COPY2]], implicit $exec
+ ; CHECK45-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY11]], [[REG_SEQUENCE1]], implicit $exec
; CHECK45-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 28
; CHECK45-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_5]]
; CHECK45-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY12]], [[COPY5]], implicit $exec
; CHECK45-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[V_LSHRREV_B64_e64_]].sub0
; CHECK45-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[V_LSHRREV_B64_e64_]].sub1
; CHECK45-NEXT: [[V_OR3_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR3_B32_e64 [[COPY14]], [[V_LSHLREV_B32_e64_]], [[V_LSHLREV_B32_e64_1]], implicit $exec
- ; CHECK45-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[V_LSHL_OR_B32_e64_]], %subreg.sub1, [[COPY13]], %subreg.sub2, [[V_OR3_B32_e64_]], %subreg.sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_128_align2 = REG_SEQUENCE [[COPY8]], %subreg.sub0, [[V_LSHL_OR_B32_e64_]], %subreg.sub1, [[COPY13]], %subreg.sub2, [[V_OR3_B32_e64_]], %subreg.sub3
; CHECK45-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
; CHECK45-NEXT: [[S_MOV_B32_6:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; CHECK45-NEXT: {{ $}}
@@ -736,11 +738,11 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[V_LSHL_OR_B32_e64_]], implicit $exec
; CHECK45-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY13]], implicit $exec
; CHECK45-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[V_OR3_B32_e64_]], implicit $exec
- ; CHECK45-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; CHECK45-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub0_sub1
- ; CHECK45-NEXT: [[COPY17:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE2]].sub2_sub3
- ; CHECK45-NEXT: [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub0_sub1
- ; CHECK45-NEXT: [[COPY19:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]].sub2_sub3
+ ; CHECK45-NEXT: [[REG_SEQUENCE4:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; CHECK45-NEXT: [[COPY16:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]].sub0_sub1
+ ; CHECK45-NEXT: [[COPY17:%[0-9]+]]:vreg_64_align2 = COPY [[REG_SEQUENCE3]].sub2_sub3
+ ; CHECK45-NEXT: [[COPY18:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE4]].sub0_sub1
+ ; CHECK45-NEXT: [[COPY19:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE4]].sub2_sub3
; CHECK45-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY18]], [[COPY16]], implicit $exec
; CHECK45-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY19]], [[COPY17]], implicit $exec
; CHECK45-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
@@ -749,7 +751,7 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
; CHECK45-NEXT: bb.3:
; CHECK45-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; CHECK45-NEXT: {{ $}}
- ; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN [[COPY15]], [[REG_SEQUENCE3]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.rsrc, align 1, addrspace 8)
+ ; CHECK45-NEXT: [[BUFFER_LOAD_DWORD_VBUFFER_IDXEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_VBUFFER_IDXEN [[COPY15]], [[REG_SEQUENCE4]], $sgpr_null, 0, 0, 0, implicit $exec :: (dereferenceable load (f32) from %ir.rsrc, align 1, addrspace 8)
; CHECK45-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; CHECK45-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; CHECK45-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
index 28d1241adb67a..525d43ecbd6e1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
@@ -47,8 +47,9 @@ define amdgpu_ps half @raw_buffer_load_format_f16__sgpr_rsrc__vgpr_voffset__sgpr
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -347,8 +348,9 @@ define amdgpu_ps half @raw_buffer_load_format_f16__vgpr_rsrc__sgpr_voffset__vgpr
; GFX12-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.5:
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY11]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -476,8 +478,10 @@ define amdgpu_ps half @raw_buffer_load_format_f16__sgpr_rsrc__sgpr_voffset__sgpr
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]], implicit $exec
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[V_READFIRSTLANE_B32_]]
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.format.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
index 63bce3c9706b2..3e7c1fedea977 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.ll
@@ -808,8 +808,9 @@ define amdgpu_ps half @raw_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffse
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -1149,8 +1150,9 @@ define amdgpu_ps half @raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffse
; GFX1200-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX1200-NEXT: {{ $}}
; GFX1200-NEXT: bb.5:
- ; GFX1200-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1200-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1200-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; GFX1250-LABEL: name: raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX1250: bb.1 (%ir-block.0):
@@ -1196,8 +1198,9 @@ define amdgpu_ps half @raw_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffse
; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.5:
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.buffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
index 02179b115562a..7ea242c7f66e0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.ll
@@ -684,8 +684,9 @@ define amdgpu_ps half @raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_so
; GFX1250-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX1250-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16) from %ir.rsrc, align 1, addrspace 8)
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1250-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -1007,8 +1008,9 @@ define amdgpu_ps half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_so
; GFX1250-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX1250-NEXT: {{ $}}
; GFX1250-NEXT: bb.5:
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]]
- ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
index 358a570c1dc15..ec6e714754c70 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
@@ -46,8 +46,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret half %val
}
@@ -345,8 +346,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-NEXT: $exec_lo = S_MOV_B32_term [[S_MOV_B32_]]
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.5:
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY11]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret half %val
}
@@ -394,8 +396,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 1, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
ret half %val
}
@@ -443,8 +446,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 2, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
ret half %val
}
@@ -492,8 +496,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 3, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
ret half %val
}
@@ -541,8 +546,9 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 4, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY6]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
ret half %val
}
@@ -593,8 +599,10 @@ define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__sgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_OFFEN]], implicit $exec
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[V_READFIRSTLANE_B32_]]
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.tbuffer.load.f16(<4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
index ce4599e7e1e05..7fd96fc64af5f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.i8.ll
@@ -36,9 +36,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffs
;
; GFX12-LABEL: name: raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -46,7 +46,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__sgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.i8(i8 %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 94, i32 0)
ret void
@@ -151,9 +153,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -161,6 +163,8 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
@@ -170,11 +174,11 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
@@ -183,7 +187,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__sgpr_soffs
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE2]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -303,9 +307,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; GFX12-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -313,6 +317,8 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
@@ -322,11 +328,11 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
@@ -338,7 +344,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__vgpr_voffset__vgpr_soffs
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -460,9 +466,9 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-LABEL: name: raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
@@ -470,6 +476,8 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
@@ -480,11 +488,11 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
@@ -496,7 +504,7 @@ define amdgpu_ps void @raw_tbuffer_store_i8__vgpr_rsrc__sgpr_voffset__vgpr_soffs
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[COPY]], [[COPY7]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (i8), addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
index 8202e72aab96b..86e0b2fe9bb16 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.ll
@@ -996,8 +996,9 @@ define amdgpu_ps half @struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voff
; GFX1200-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX1200-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX1200-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1200-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
- ; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1200-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]].lo16
+ ; GFX1200-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX1200-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; GFX1250-LABEL: name: struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; GFX1250: bb.1 (%ir-block.0):
@@ -1013,8 +1014,9 @@ define amdgpu_ps half @struct_buffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voff
; GFX1250-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX1250-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64_align2 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX1250-NEXT: [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_USHORT_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX1250-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]]
- ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX1250-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_BOTHEN]].lo16
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX1250-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.struct.buffer.load.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
index f418ec0d41c55..b4f2d503f67a2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
@@ -5,23 +5,6 @@
; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=UNPACKED %s
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16) from %ir.rsrc, align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
@@ -160,24 +143,6 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__sgpr_rsrc__vgpr_vindex__
}
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0(ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16) from %ir.rsrc, align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
@@ -336,23 +301,6 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
}
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset.base, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (f16) from %ir.rsrc, align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
index a568c9b324320..413afff6d3b20 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
@@ -6,23 +6,6 @@
; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=UNPACKED %s
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; GFX12-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
@@ -37,8 +20,9 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; UNPACKED: bb.1 (%ir-block.0):
@@ -215,24 +199,6 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__sgpr_rsrc__vgpr_vindex__
}
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0(<4 x i32> inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; GFX12-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
@@ -248,8 +214,9 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY4]], %subreg.sub1
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_vindex0
; UNPACKED: bb.1 (%ir-block.0):
@@ -466,23 +433,6 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
}
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset.base, i32 inreg %soffset) {
- ; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
- ; PACKED: bb.1 (%ir-block.0):
- ; PACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
- ; PACKED-NEXT: {{ $}}
- ; PACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; PACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; PACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; PACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; PACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; PACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; PACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; PACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; PACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
- ; PACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; PACKED-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_BOTHEN]]
- ; PACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
- ;
; GFX12-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
@@ -497,8 +447,9 @@ define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_vof
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 78, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[TBUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
;
; UNPACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset_voffset_add4095
; UNPACKED: bb.1 (%ir-block.0):
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
index 540be44df1bfd..af6bfc3fc8559 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
@@ -698,9 +698,10 @@ define <12 x i8> @v_load_constant_v12i8_align8(ptr addrspace(4) %ptr) {
; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v9, 8, v2
; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v10, 16, v2
; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX12-UNALIGNED-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v13
-; GFX12-UNALIGNED-NEXT: v_mov_b32_e32 v8, v2
-; GFX12-UNALIGNED-NEXT: v_mov_b32_e32 v2, v12
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v1.l, v13.l
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v2.l, v12.l
; GFX12-UNALIGNED-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-NOUNALIGNED-LABEL: v_load_constant_v12i8_align8:
@@ -721,9 +722,10 @@ define <12 x i8> @v_load_constant_v12i8_align8(ptr addrspace(4) %ptr) {
; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v9, 8, v2
; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v10, 16, v2
; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX12-NOUNALIGNED-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v13
-; GFX12-NOUNALIGNED-NEXT: v_mov_b32_e32 v8, v2
-; GFX12-NOUNALIGNED-NEXT: v_mov_b32_e32 v2, v12
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v1.l, v13.l
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v2.l, v12.l
; GFX12-NOUNALIGNED-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_load_constant_v12i8_align8:
@@ -736,10 +738,12 @@ define <12 x i8> @v_load_constant_v12i8_align8(ptr addrspace(4) %ptr) {
; GFX1250-NEXT: v_dual_lshrrev_b32 v3, 24, v0 :: v_dual_lshrrev_b32 v5, 8, v1
; GFX1250-NEXT: v_dual_lshrrev_b32 v6, 16, v1 :: v_dual_lshrrev_b32 v7, 24, v1
; GFX1250-NEXT: v_dual_lshrrev_b32 v9, 8, v2 :: v_dual_lshrrev_b32 v10, 16, v2
-; GFX1250-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_lshrrev_b32 v11, 24, v2
+; GFX1250-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX1250-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX1250-NEXT: v_mov_b16_e32 v8.l, v2.l
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v1, v13
-; GFX1250-NEXT: v_mov_b32_e32 v2, v12
+; GFX1250-NEXT: v_mov_b16_e32 v1.l, v13.l
+; GFX1250-NEXT: v_mov_b16_e32 v2.l, v12.l
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX9-LABEL: v_load_constant_v12i8_align8:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll
index 0a12cd16159b0..1e23985c424d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/or.ll
@@ -33,11 +33,23 @@ define i16 @v_or_i16(i16 %num, i16 %den) {
; GCN-NEXT: v_or_b32_e32 v0, v0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_or_i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_or_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_or_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_or_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: v_or_i16:
; GFX12: ; %bb.0:
@@ -46,7 +58,7 @@ define i16 @v_or_i16(i16 %num, i16 %den) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = or i16 %num, %den
ret i16 %result
@@ -1109,6 +1121,3 @@ define <2 x i256> @v_or_v2i256(<2 x i256> %a, <2 x i256> %b) {
ret <2 x i256> %or
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-FAKE16: {{.*}}
-; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index dc11e84be1abb..f7a814b5d0cdb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -404,10 +404,8 @@ define i16 @v_saddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-TRUE16-LABEL: v_saddsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_i16 v0, v0, v1 clamp
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index d797b0fe33673..66ba34d4b187f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -404,10 +404,8 @@ define i16 @v_ssubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-TRUE16-LABEL: v_ssubsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 163698d594dc5..eb9bd4fa1fb9f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -330,10 +330,8 @@ define i16 @v_uaddsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-TRUE16-LABEL: v_uaddsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v0, v1 clamp
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index 2d3abaf51da69..9a401ab1df3aa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -324,10 +324,8 @@ define i16 @v_usubsat_v2i8(i16 %lhs.arg, i16 %rhs.arg) {
; GFX11-TRUE16-LABEL: v_usubsat_v2i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll
index 41e66b9122086..7ae9ae02a961d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/xor.ll
@@ -33,11 +33,23 @@ define i16 @v_xor_i16(i16 %num, i16 %den) {
; GCN-NEXT: v_xor_b32_e32 v0, v0, v1
; GCN-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_xor_i16:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_xor_i16:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: v_xor_i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: v_xor_i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: v_xor_i16:
; GFX12: ; %bb.0:
@@ -46,7 +58,7 @@ define i16 @v_xor_i16(i16 %num, i16 %den) {
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX12-NEXT: v_xor_b16 v0.l, v0.l, v1.l
; GFX12-NEXT: s_setpc_b64 s[30:31]
%result = xor i16 %num, %den
ret i16 %result
@@ -1109,6 +1121,3 @@ define <2 x i256> @v_xor_v2i256(<2 x i256> %a, <2 x i256> %b) {
ret <2 x i256> %xor
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11-FAKE16: {{.*}}
-; GFX11-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 951c1add52339..991071ac55bc5 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -2929,93 +2929,94 @@ define <4 x i32> @bitcast_v8bf16_to_v4i32(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v4i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v10 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
-; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: .LBB22_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v4i32:
@@ -3797,57 +3798,64 @@ define <16 x i8> @bitcast_v4i32_to_v16i8(<4 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB24_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v19
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v17, 3, v17
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 3, v16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v18
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB24_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i32_to_v16i8:
@@ -4160,85 +4168,76 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s18, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB25_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s8 :: v_dual_mov_b32 v0, s0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v12, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s6
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB25_3
; GFX11-TRUE16-NEXT: .LBB25_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s18, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
; GFX11-TRUE16-NEXT: .LBB25_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s18, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB25_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
; GFX11-TRUE16-NEXT: s_add_i32 s3, s3, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s4 :: v_dual_mov_b32 v12, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s3, 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s6 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v15, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v13, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s5 :: v_dual_mov_b32 v9, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v7, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
; GFX11-TRUE16-NEXT: .LBB25_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, s9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, s8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i32_to_v16i8_scalar:
@@ -4636,10 +4635,10 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v4i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -4652,8 +4651,8 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -4663,10 +4662,10 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -4684,11 +4683,11 @@ define <4 x i32> @bitcast_v16i8_to_v4i32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -5136,168 +5135,84 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB27_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v4i32_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB27_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
-; GFX11-TRUE16-NEXT: s_branch .LBB27_3
-; GFX11-TRUE16-NEXT: .LBB27_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB27_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB27_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: .LBB27_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v4i32_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB27_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-FAKE16-NEXT: s_branch .LBB27_3
-; GFX11-FAKE16-NEXT: .LBB27_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB27_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB27_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-FAKE16-NEXT: .LBB27_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v16i8_to_v4i32_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB27_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-NEXT: s_branch .LBB27_3
+; GFX11-NEXT: .LBB27_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB27_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: s_add_i32 s24, s24, 3
+; GFX11-NEXT: s_add_i32 s26, s26, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-NEXT: .LBB27_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7851,93 +7766,94 @@ define <4 x float> @bitcast_v8bf16_to_v4f32(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v4f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v10 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
-; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: .LBB46_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v4f32:
@@ -8719,55 +8635,62 @@ define <16 x i8> @bitcast_v4f32_to_v16i8(<4 x float> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB48_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v16, 1.0, v16
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v10, 1.0, v10
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v16, 1.0, v16
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v17, 1.0, v17 :: v_dual_add_f32 v18, 1.0, v18
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f32_to_v16i8:
@@ -9098,55 +9021,41 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v5, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, s6
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB49_3
; GFX11-TRUE16-NEXT: .LBB49_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
@@ -9157,21 +9066,34 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v7, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v15, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s10 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f32_to_v16i8_scalar:
@@ -9578,10 +9500,10 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v4f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -9594,8 +9516,8 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -9605,10 +9527,10 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -9626,11 +9548,11 @@ define <4 x float> @bitcast_v16i8_to_v4f32(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -10078,168 +10000,84 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v4f32_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
-; GFX11-TRUE16-NEXT: s_branch .LBB51_3
-; GFX11-TRUE16-NEXT: .LBB51_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB51_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: .LBB51_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v4f32_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-FAKE16-NEXT: s_branch .LBB51_3
-; GFX11-FAKE16-NEXT: .LBB51_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB51_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB51_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-FAKE16-NEXT: .LBB51_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v16i8_to_v4f32_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB51_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-NEXT: s_branch .LBB51_3
+; GFX11-NEXT: .LBB51_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB51_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: s_add_i32 s24, s24, 3
+; GFX11-NEXT: s_add_i32 s26, s26, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-NEXT: .LBB51_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -12361,93 +12199,94 @@ define <2 x i64> @bitcast_v8bf16_to_v2i64(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v2i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v10 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
-; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: .LBB66_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v2i64:
@@ -13229,28 +13068,36 @@ define <16 x i8> @bitcast_v2i64_to_v16i8(<2 x i64> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB68_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_4
@@ -13258,28 +13105,27 @@ define <16 x i8> @bitcast_v2i64_to_v16i8(<2 x i64> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_co_u32 v16, vcc_lo, v16, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v17, null, 0, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB68_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2i64_to_v16i8:
@@ -13592,87 +13438,76 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s18, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB69_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s8 :: v_dual_mov_b32 v0, s0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v12, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s6
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB69_3
; GFX11-TRUE16-NEXT: .LBB69_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s18, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
; GFX11-TRUE16-NEXT: .LBB69_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s18, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB69_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, 0
; GFX11-TRUE16-NEXT: s_add_u32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_addc_u32 s3, s3, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v15, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v9, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v5, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s2 :: v_dual_mov_b32 v6, s3
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
; GFX11-TRUE16-NEXT: .LBB69_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, s9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, s8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2i64_to_v16i8_scalar:
@@ -14070,10 +13905,10 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v2i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -14086,8 +13921,8 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -14097,10 +13932,10 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -14118,11 +13953,11 @@ define <2 x i64> @bitcast_v16i8_to_v2i64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2
; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -14570,168 +14405,84 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v2i64_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
-; GFX11-TRUE16-NEXT: s_branch .LBB71_3
-; GFX11-TRUE16-NEXT: .LBB71_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB71_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB71_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: .LBB71_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v2i64_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-FAKE16-NEXT: s_branch .LBB71_3
-; GFX11-FAKE16-NEXT: .LBB71_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB71_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB71_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-FAKE16-NEXT: .LBB71_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v16i8_to_v2i64_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-NEXT: s_branch .LBB71_3
+; GFX11-NEXT: .LBB71_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB71_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: s_add_i32 s24, s24, 3
+; GFX11-NEXT: s_add_i32 s26, s26, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-NEXT: .LBB71_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -16437,93 +16188,94 @@ define <2 x double> @bitcast_v8bf16_to_v2f64(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v2f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v10 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v3, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v11, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.h
-; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: .LBB82_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v2f64:
@@ -17303,54 +17055,61 @@ define <16 x i8> @bitcast_v2f64_to_v16i8(<2 x double> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB84_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[16:17], v[16:17], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], v[18:19], 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB84_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2f64_to_v16i8:
@@ -17674,78 +17433,76 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB85_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s8 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v1, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v16, s6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s0
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB85_3
; GFX11-TRUE16-NEXT: .LBB85_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
; GFX11-TRUE16-NEXT: .LBB85_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB85_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[11:12], s[2:3], 1.0
; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], s[0:1], 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[17:18], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: .LBB85_5: ; %end
+; GFX11-TRUE16-NEXT: s_branch .LBB85_6
+; GFX11-TRUE16-NEXT: .LBB85_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v1, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, s15
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v14, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s6 :: v_dual_mov_b32 v16, s12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s14 :: v_dual_mov_b32 v6, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s11 :: v_dual_mov_b32 v8, s9
+; GFX11-TRUE16-NEXT: .LBB85_6: ; %end
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2f64_to_v16i8_scalar:
@@ -18149,10 +17906,10 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v2f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -18165,8 +17922,8 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -18176,10 +17933,10 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -18197,11 +17954,11 @@ define <2 x double> @bitcast_v16i8_to_v2f64(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2
; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -18649,168 +18406,84 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: .LBB87_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v2f64_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB87_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
-; GFX11-TRUE16-NEXT: s_branch .LBB87_3
-; GFX11-TRUE16-NEXT: .LBB87_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB87_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB87_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: .LBB87_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v2f64_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB87_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-FAKE16-NEXT: s_branch .LBB87_3
-; GFX11-FAKE16-NEXT: .LBB87_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB87_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB87_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-FAKE16-NEXT: .LBB87_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v16i8_to_v2f64_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB87_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-NEXT: s_branch .LBB87_3
+; GFX11-NEXT: .LBB87_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB87_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: s_add_i32 s24, s24, 3
+; GFX11-NEXT: s_add_i32 s26, s26, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-NEXT: .LBB87_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -20131,71 +19804,89 @@ define <8 x i16> @bitcast_v8bf16_to_v8i16(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v12, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v11, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v1, v7, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v6, v8, v9 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v13 :: v_dual_and_b32 v2, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v11, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v12, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v10.l
; GFX11-TRUE16-NEXT: .LBB94_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -21038,57 +20729,64 @@ define <16 x i8> @bitcast_v8i16_to_v16i8(<8 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB96_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v17, v17, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v16, v16, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB96_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8i16_to_v16i8:
@@ -21453,55 +21151,41 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v5, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, s6
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB97_3
; GFX11-TRUE16-NEXT: .LBB97_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
; GFX11-TRUE16-NEXT: .LBB97_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
@@ -21512,21 +21196,34 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: .LBB97_5: ; %end
+; GFX11-TRUE16-NEXT: s_branch .LBB97_6
+; GFX11-TRUE16-NEXT: .LBB97_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v7, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v15, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s10 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB97_6: ; %end
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8i16_to_v16i8_scalar:
@@ -21954,10 +21651,10 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -21970,8 +21667,8 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -21981,10 +21678,10 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -22002,11 +21699,11 @@ define <8 x i16> @bitcast_v16i8_to_v8i16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2
; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -22481,168 +22178,84 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8i16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
-; GFX11-TRUE16-NEXT: s_branch .LBB99_3
-; GFX11-TRUE16-NEXT: .LBB99_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB99_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB99_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: .LBB99_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v8i16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-FAKE16-NEXT: s_branch .LBB99_3
-; GFX11-FAKE16-NEXT: .LBB99_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB99_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB99_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-FAKE16-NEXT: .LBB99_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v16i8_to_v8i16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB99_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-NEXT: s_branch .LBB99_3
+; GFX11-NEXT: .LBB99_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB99_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: s_add_i32 s24, s24, 3
+; GFX11-NEXT: s_add_i32 s26, s26, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-NEXT: .LBB99_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -23306,90 +22919,97 @@ define <8 x half> @bitcast_v8bf16_to_v8f16(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v8f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_mov_b32 v6, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_and_b32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v7 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v1, v7, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v7, v12 :: v_dual_and_b32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v13, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v9, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.h
-; GFX11-TRUE16-NEXT: .LBB102_4: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v7.l
+; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v6 :: v_dual_mov_b32 v3, v7
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v8f16:
@@ -24258,57 +23878,64 @@ define <16 x i8> @bitcast_v8f16_to_v16i8(<8 x half> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB104_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v17, 0x200, v17 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB104_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8f16_to_v16i8:
@@ -24693,55 +24320,41 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB105_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v5, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, s6
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB105_3
; GFX11-TRUE16-NEXT: .LBB105_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
; GFX11-TRUE16-NEXT: .LBB105_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
@@ -24752,21 +24365,34 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: .LBB105_5: ; %end
+; GFX11-TRUE16-NEXT: s_branch .LBB105_6
+; GFX11-TRUE16-NEXT: .LBB105_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v7, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v15, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v13, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s10 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB105_6: ; %end
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8f16_to_v16i8_scalar:
@@ -25194,10 +24820,10 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -25210,8 +24836,8 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -25221,10 +24847,10 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -25242,11 +24868,11 @@ define <8 x half> @bitcast_v16i8_to_v8f16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2
; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -25721,168 +25347,84 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8f16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
-; GFX11-TRUE16-NEXT: s_branch .LBB107_3
-; GFX11-TRUE16-NEXT: .LBB107_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB107_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB107_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: .LBB107_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v8f16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-FAKE16-NEXT: s_branch .LBB107_3
-; GFX11-FAKE16-NEXT: .LBB107_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB107_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB107_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-FAKE16-NEXT: .LBB107_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v16i8_to_v8f16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB107_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-NEXT: s_branch .LBB107_3
+; GFX11-NEXT: .LBB107_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB107_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: s_add_i32 s24, s24, 3
+; GFX11-NEXT: s_add_i32 s26, s26, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-NEXT: .LBB107_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -26275,132 +25817,137 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8bf16_to_v16i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v3 :: v_dual_mov_b32 v16, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v18, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v4
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[2:3]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[0:1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.h, v3.h
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v18.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v4, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v8, v10 :: v_dual_and_b32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v14, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v13, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v16, v11, v12 :: v_dual_add_f32 v1, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v8, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v6.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v0, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v11, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v7, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v22, v5, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v9, v7 :: v_dual_and_b32 v9, 0xffff0000, v16
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v12.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v21, v0, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v8.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v22
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v22
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[18:19], 24, v[21:22]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v21
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v2.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v14.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v10.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v17.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v21.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v22.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v16i8:
@@ -26968,150 +26515,154 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v5, s6
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, s6
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB109_3
; GFX11-TRUE16-NEXT: .LBB109_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
; GFX11-TRUE16-NEXT: .LBB109_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB109_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s0
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s3
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s4
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s3
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s4
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v10, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s3, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, v11, v10
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v8, v7
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x7fff, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v9, v7
; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s2, 16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v11, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, v13, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v10, v12
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v6.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v3, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v14.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: .LBB109_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[17:18]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v17
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v10.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v18
+; GFX11-TRUE16-NEXT: s_branch .LBB109_6
+; GFX11-TRUE16-NEXT: .LBB109_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v13, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s18 :: v_dual_mov_b32 v5, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v9, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v15, s11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v1, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v7, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v3, s4
+; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v16i8_scalar:
@@ -27615,10 +27166,10 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v16
@@ -27631,8 +27182,8 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v20, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v18, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v19, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v17, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
@@ -27642,10 +27193,10 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -27663,11 +27214,11 @@ define <8 x bfloat> @bitcast_v16i8_to_v8bf16(<16 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2
; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v19.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v17.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v20.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
@@ -28138,168 +27689,84 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8bf16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
-; GFX11-TRUE16-NEXT: s_branch .LBB111_3
-; GFX11-TRUE16-NEXT: .LBB111_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB111_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB111_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
-; GFX11-TRUE16-NEXT: .LBB111_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v8bf16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-FAKE16-NEXT: s_branch .LBB111_3
-; GFX11-FAKE16-NEXT: .LBB111_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB111_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB111_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-FAKE16-NEXT: .LBB111_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v16i8_to_v8bf16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB111_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-NEXT: s_branch .LBB111_3
+; GFX11-NEXT: .LBB111_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB111_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: s_add_i32 s24, s24, 3
+; GFX11-NEXT: s_add_i32 s26, s26, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-NEXT: .LBB111_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
index 6df1fd4477827..e34ffdf0fdff9 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
@@ -50,23 +50,15 @@ define half @bitcast_i16_to_f16(i16 %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_i16_to_f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB0_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v0.l, 3
-; GFX11-TRUE16-NEXT: .LBB0_4: ; %end
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, 3
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i16_to_f16:
@@ -169,22 +161,20 @@ define inreg half @bitcast_i16_to_f16_scalar(i16 inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB1_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB1_3
-; GFX11-TRUE16-NEXT: .LBB1_2:
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: .LBB1_3: ; %Flow
+; GFX11-TRUE16-NEXT: .LBB1_2: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e64 v0, s0, 3
-; GFX11-TRUE16-NEXT: .LBB1_5: ; %end
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: .LBB1_4: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i16_to_f16_scalar:
@@ -281,23 +271,15 @@ define i16 @bitcast_f16_to_i16(half %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_f16_to_i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB2_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, 0x200, v0.l
-; GFX11-TRUE16-NEXT: .LBB2_4: ; %end
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, 0x200, v0.l
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_i16:
@@ -408,22 +390,20 @@ define inreg i16 @bitcast_f16_to_i16_scalar(half inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB3_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB3_3
-; GFX11-TRUE16-NEXT: .LBB3_2:
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB3_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: .LBB3_3: ; %Flow
+; GFX11-TRUE16-NEXT: .LBB3_2: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB3_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB3_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, 0x200, s0
-; GFX11-TRUE16-NEXT: .LBB3_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB3_4:
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_i16_scalar:
@@ -510,23 +490,15 @@ define bfloat @bitcast_i16_to_bf16(i16 %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_i16_to_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB4_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v0.l, 3
-; GFX11-TRUE16-NEXT: .LBB4_4: ; %end
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, 3
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i16_to_bf16:
@@ -633,22 +605,20 @@ define inreg bfloat @bitcast_i16_to_bf16_scalar(i16 inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB5_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB5_3
-; GFX11-TRUE16-NEXT: .LBB5_2:
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: .LBB5_3: ; %Flow
+; GFX11-TRUE16-NEXT: .LBB5_2: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e64 v0, s0, 3
-; GFX11-TRUE16-NEXT: .LBB5_5: ; %end
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: .LBB5_4: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i16_to_bf16_scalar:
@@ -759,61 +729,30 @@ define i16 @bitcast_bf16_to_i16(bfloat %a, i32 %b) #0 {
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_bf16_to_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_hi16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB6_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: .LBB6_4: ; %end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_bf16_to_i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-FAKE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execz .LBB6_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: .LBB6_2: ; %end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_bf16_to_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: v_cmpx_ne_u32_e32 0, v1
+; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT: s_cbranch_execz .LBB6_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: .LBB6_2: ; %end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -925,20 +864,16 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB7_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB7_3
-; GFX11-TRUE16-NEXT: .LBB7_2:
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: .LBB7_3: ; %Flow
+; GFX11-TRUE16-NEXT: .LBB7_2: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -951,7 +886,9 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: .LBB7_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB7_4:
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_bf16_to_i16_scalar:
@@ -1056,23 +993,15 @@ define bfloat @bitcast_f16_to_bf16(half %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_f16_to_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB8_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f16_e32 v1.l, 0x200, v0.l
-; GFX11-TRUE16-NEXT: .LBB8_4: ; %end
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f16_e32 v0.l, 0x200, v0.l
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_bf16:
@@ -1187,22 +1116,20 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB9_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB9_3
-; GFX11-TRUE16-NEXT: .LBB9_2:
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB9_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: .LBB9_3: ; %Flow
+; GFX11-TRUE16-NEXT: .LBB9_2: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB9_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB9_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, 0x200, s0
-; GFX11-TRUE16-NEXT: .LBB9_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB9_4:
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_bf16_scalar:
@@ -1313,61 +1240,30 @@ define half @bitcast_bf16_to_f16(bfloat %a, i32 %b) #0 {
; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_bf16_to_f16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_hi16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: .LBB10_4: ; %end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_bf16_to_f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v1
-; GFX11-FAKE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execz .LBB10_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: .LBB10_2: ; %end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_bf16_to_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-NEXT: v_cmpx_ne_u32_e32 0, v1
+; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-NEXT: s_cbranch_execz .LBB10_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: .LBB10_2: ; %end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -1479,20 +1375,16 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB11_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-TRUE16-NEXT: s_branch .LBB11_3
-; GFX11-TRUE16-NEXT: .LBB11_2:
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_2
+; GFX11-TRUE16-NEXT: ; %bb.1:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: .LBB11_3: ; %Flow
+; GFX11-TRUE16-NEXT: .LBB11_2: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_4
+; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -1505,7 +1397,9 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: .LBB11_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB11_4:
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_bf16_to_f16_scalar:
@@ -1557,5 +1451,3 @@ end:
}
attributes #0 = { nounwind }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 5bed5d26c6c02..7522d1b697b33 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -1435,40 +1435,36 @@ define i32 @bitcast_v2bf16_to_i32(<2 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2bf16_to_i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: .LBB14_4: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB14_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_i32:
@@ -2173,28 +2169,33 @@ define <4 x i8> @bitcast_i32_to_v4i8(i32 %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_i32_to_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB20_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB20_4
+; GFX11-TRUE16-NEXT: .LBB20_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB20_3: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB20_2
+; GFX11-TRUE16-NEXT: .LBB20_4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i32_to_v4i8:
@@ -2349,38 +2350,35 @@ define inreg <4 x i8> @bitcast_i32_to_v4i8_scalar(i32 inreg %a, i32 inreg %b) #0
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB21_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB21_3
; GFX11-TRUE16-NEXT: .LBB21_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr1
; GFX11-TRUE16-NEXT: .LBB21_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB21_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
; GFX11-TRUE16-NEXT: .LBB21_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i32_to_v4i8_scalar:
@@ -2785,88 +2783,46 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v4i8_to_i32_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB23_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB23_3
-; GFX11-TRUE16-NEXT: .LBB23_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: .LBB23_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: .LBB23_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v4i8_to_i32_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB23_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB23_3
-; GFX11-FAKE16-NEXT: .LBB23_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-FAKE16-NEXT: .LBB23_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: .LBB23_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v4i8_to_i32_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB23_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_branch .LBB23_3
+; GFX11-NEXT: .LBB23_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB23_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: .LBB23_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -3984,40 +3940,36 @@ define float @bitcast_v2bf16_to_f32(<2 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2bf16_to_f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB34_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB34_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: .LBB34_4: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB34_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_f32:
@@ -4725,28 +4677,33 @@ define <4 x i8> @bitcast_f32_to_v4i8(float %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_f32_to_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 1.0, v2
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_4
+; GFX11-TRUE16-NEXT: .LBB40_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB40_3: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB40_2
+; GFX11-TRUE16-NEXT: .LBB40_4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 1.0, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f32_to_v4i8:
@@ -4900,74 +4857,39 @@ define inreg <4 x i8> @bitcast_f32_to_v4i8_scalar(float inreg %a, i32 inreg %b)
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_f32_to_v4i8_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB41_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX11-TRUE16-NEXT: s_branch .LBB41_3
-; GFX11-TRUE16-NEXT: .LBB41_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: .LBB41_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB41_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, s0, 1.0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: .LBB41_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_f32_to_v4i8_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB41_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-FAKE16-NEXT: s_branch .LBB41_3
-; GFX11-FAKE16-NEXT: .LBB41_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr1
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-FAKE16-NEXT: .LBB41_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB41_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, s0, 1.0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB41_5:
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_f32_to_v4i8_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr1
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-NEXT: .LBB41_5:
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5337,88 +5259,46 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; GFX9-NEXT: .LBB43_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v4i8_to_f32_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB43_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB43_3
-; GFX11-TRUE16-NEXT: .LBB43_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: .LBB43_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB43_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: .LBB43_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v4i8_to_f32_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB43_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB43_3
-; GFX11-FAKE16-NEXT: .LBB43_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-FAKE16-NEXT: .LBB43_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB43_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: .LBB43_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v4i8_to_f32_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB43_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_branch .LBB43_3
+; GFX11-NEXT: .LBB43_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB43_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: .LBB43_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -6156,24 +6036,27 @@ define <2 x i16> @bitcast_v2bf16_to_v2i16(<2 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: .LBB50_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -6943,28 +6826,33 @@ define <4 x i8> @bitcast_v2i16_to_v4i8(<2 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2i16_to_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v2, v2, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_4
+; GFX11-TRUE16-NEXT: .LBB56_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB56_3: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB56_2
+; GFX11-TRUE16-NEXT: .LBB56_4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v0, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2i16_to_v4i8:
@@ -7129,74 +7017,39 @@ define inreg <4 x i8> @bitcast_v2i16_to_v4i8_scalar(<2 x i16> inreg %a, i32 inre
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v2i16_to_v4i8_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB57_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX11-TRUE16-NEXT: s_branch .LBB57_3
-; GFX11-TRUE16-NEXT: .LBB57_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: .LBB57_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: .LBB57_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v2i16_to_v4i8_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB57_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-FAKE16-NEXT: s_branch .LBB57_3
-; GFX11-FAKE16-NEXT: .LBB57_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr1
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-FAKE16-NEXT: .LBB57_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB57_5:
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v2i16_to_v4i8_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB57_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-NEXT: s_branch .LBB57_3
+; GFX11-NEXT: .LBB57_2:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr1
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: .LBB57_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-NEXT: .LBB57_5:
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7572,88 +7425,46 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; GFX9-NEXT: .LBB59_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v4i8_to_v2i16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB59_3
-; GFX11-TRUE16-NEXT: .LBB59_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: .LBB59_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: .LBB59_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v4i8_to_v2i16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB59_3
-; GFX11-FAKE16-NEXT: .LBB59_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-FAKE16-NEXT: .LBB59_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: .LBB59_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v4i8_to_v2i16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB59_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_branch .LBB59_3
+; GFX11-NEXT: .LBB59_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB59_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: .LBB59_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7992,40 +7803,36 @@ define <2 x half> @bitcast_v2bf16_to_v2f16(<2 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2bf16_to_v2f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB62_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB62_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: .LBB62_4: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB62_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_v2f16:
@@ -8804,28 +8611,33 @@ define <4 x i8> @bitcast_v2f16_to_v4i8(<2 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2f16_to_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB68_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v2, 0x200, v2 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB68_4
+; GFX11-TRUE16-NEXT: .LBB68_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB68_3: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_2
+; GFX11-TRUE16-NEXT: .LBB68_4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v0, 0x200, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2f16_to_v4i8:
@@ -8993,74 +8805,39 @@ define inreg <4 x i8> @bitcast_v2f16_to_v4i8_scalar(<2 x half> inreg %a, i32 inr
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v2f16_to_v4i8_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB69_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX11-TRUE16-NEXT: s_branch .LBB69_3
-; GFX11-TRUE16-NEXT: .LBB69_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: .LBB69_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB69_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: .LBB69_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v2f16_to_v4i8_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB69_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-FAKE16-NEXT: s_branch .LBB69_3
-; GFX11-FAKE16-NEXT: .LBB69_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr1
-; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-FAKE16-NEXT: .LBB69_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB69_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-FAKE16-NEXT: .LBB69_5:
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v2f16_to_v4i8_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-NEXT: s_branch .LBB69_3
+; GFX11-NEXT: .LBB69_2:
+; GFX11-NEXT: s_mov_b32 s3, -1
+; GFX11-NEXT: ; implicit-def: $sgpr4
+; GFX11-NEXT: ; implicit-def: $sgpr1
+; GFX11-NEXT: ; implicit-def: $sgpr2
+; GFX11-NEXT: .LBB69_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-NEXT: .LBB69_5:
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9436,88 +9213,46 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v4i8_to_v2f16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB71_3
-; GFX11-TRUE16-NEXT: .LBB71_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: .LBB71_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB71_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: .LBB71_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v4i8_to_v2f16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB71_3
-; GFX11-FAKE16-NEXT: .LBB71_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-FAKE16-NEXT: .LBB71_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB71_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: .LBB71_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v4i8_to_v2f16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_branch .LBB71_3
+; GFX11-NEXT: .LBB71_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB71_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: .LBB71_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9635,40 +9370,36 @@ define <1 x i32> @bitcast_v2bf16_to_v1i32(<2 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2bf16_to_v1i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB72_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB72_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: .LBB72_4: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: .LBB72_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_v1i32:
@@ -10270,50 +10001,51 @@ define <4 x i8> @bitcast_v2bf16_to_v4i8(<2 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v2bf16_to_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.h
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB76_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB76_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB76_4
+; GFX11-TRUE16-NEXT: .LBB76_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB76_3: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB76_2
+; GFX11-TRUE16-NEXT: .LBB76_4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: .LBB76_4: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_v4i8:
@@ -10531,31 +10263,27 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB77_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB77_3
; GFX11-TRUE16-NEXT: .LBB77_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr1
; GFX11-TRUE16-NEXT: .LBB77_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB77_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff0000
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xffff0000
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -10573,14 +10301,19 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: .LBB77_5: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB77_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v1, s3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_v4i8_scalar:
@@ -11007,88 +10740,46 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; GFX9-NEXT: .LBB79_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v4i8_to_v2bf16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB79_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB79_3
-; GFX11-TRUE16-NEXT: .LBB79_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: .LBB79_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB79_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: .LBB79_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v4i8_to_v2bf16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB79_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB79_3
-; GFX11-FAKE16-NEXT: .LBB79_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-FAKE16-NEXT: .LBB79_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB79_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: .LBB79_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v4i8_to_v2bf16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB79_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_branch .LBB79_3
+; GFX11-NEXT: .LBB79_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB79_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: .LBB79_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11200,28 +10891,33 @@ define <4 x i8> @bitcast_v1i32_to_v4i8(<1 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v1i32_to_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB80_3
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB80_4
+; GFX11-TRUE16-NEXT: .LBB80_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-NEXT: .LBB80_3: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB80_2
+; GFX11-TRUE16-NEXT: .LBB80_4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v1i32_to_v4i8:
@@ -11376,38 +11072,35 @@ define inreg <4 x i8> @bitcast_v1i32_to_v4i8_scalar(<1 x i32> inreg %a, i32 inre
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB81_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB81_3
; GFX11-TRUE16-NEXT: .LBB81_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr1
; GFX11-TRUE16-NEXT: .LBB81_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB81_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
; GFX11-TRUE16-NEXT: .LBB81_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v1i32_to_v4i8_scalar:
@@ -11812,88 +11505,46 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; GFX9-NEXT: .LBB83_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v4i8_to_v1i32_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB83_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX11-TRUE16-NEXT: s_branch .LBB83_3
-; GFX11-TRUE16-NEXT: .LBB83_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: .LBB83_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: .LBB83_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v4i8_to_v1i32_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB83_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB83_3
-; GFX11-FAKE16-NEXT: .LBB83_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-FAKE16-NEXT: .LBB83_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: .LBB83_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v4i8_to_v1i32_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB83_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: s_branch .LBB83_3
+; GFX11-NEXT: .LBB83_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0
+; GFX11-NEXT: .LBB83_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: .LBB83_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index c63815397a96d..8b1ab6672ae86 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -2393,57 +2393,56 @@ define i64 @bitcast_v4bf16_to_i64(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v7 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: .LBB22_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_i64:
@@ -2690,48 +2689,46 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_nc_u32 v8, v8, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v5 :: v_dual_add_nc_u32 v5, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v5 :: v_dual_add_nc_u32 v9, v9, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB23_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -2939,36 +2936,42 @@ define <8 x i8> @bitcast_i64_to_v8i8(i64 %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_co_u32 v8, vcc_lo, v8, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v9, null, 0, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: .LBB24_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i64_to_v8i8:
@@ -3175,55 +3178,48 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB25_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB25_3
; GFX11-TRUE16-NEXT: .LBB25_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
; GFX11-TRUE16-NEXT: .LBB25_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB25_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
; GFX11-TRUE16-NEXT: .LBB25_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i64_to_v8i8_scalar:
@@ -3476,8 +3472,8 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -3490,12 +3486,12 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -3508,11 +3504,11 @@ define i64 @bitcast_v8i8_to_i64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -3790,122 +3786,62 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; GFX9-NEXT: .LBB27_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v8i8_to_i64_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB27_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
-; GFX11-TRUE16-NEXT: s_branch .LBB27_3
-; GFX11-TRUE16-NEXT: .LBB27_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-TRUE16-NEXT: .LBB27_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB27_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: .LBB27_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v8i8_to_i64_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB27_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB27_3
-; GFX11-FAKE16-NEXT: .LBB27_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-FAKE16-NEXT: .LBB27_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB27_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: .LBB27_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v8i8_to_i64_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB27_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-NEXT: s_branch .LBB27_3
+; GFX11-NEXT: .LBB27_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB27_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: .LBB27_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5934,57 +5870,56 @@ define double @bitcast_v4bf16_to_f64(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v7 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: .LBB46_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_f64:
@@ -6231,48 +6166,46 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_nc_u32 v8, v8, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v5 :: v_dual_add_nc_u32 v5, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v5 :: v_dual_add_nc_u32 v9, v9, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB47_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -6476,18 +6409,23 @@ define <8 x i8> @bitcast_f64_to_v8i8(double %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
@@ -6497,14 +6435,14 @@ define <8 x i8> @bitcast_f64_to_v8i8(double %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f64_to_v8i8:
@@ -6719,49 +6657,51 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB49_3
; GFX11-TRUE16-NEXT: .LBB49_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], s[0:1], 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: s_branch .LBB49_6
+; GFX11-TRUE16-NEXT: .LBB49_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v1, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s2 :: v_dual_mov_b32 v6, s5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v8, s3
+; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f64_to_v8i8_scalar:
@@ -7018,8 +6958,8 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -7032,12 +6972,12 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -7050,11 +6990,11 @@ define double @bitcast_v8i8_to_f64(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -7332,122 +7272,62 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v8i8_to_f64_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
-; GFX11-TRUE16-NEXT: s_branch .LBB51_3
-; GFX11-TRUE16-NEXT: .LBB51_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-TRUE16-NEXT: .LBB51_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: .LBB51_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v8i8_to_f64_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB51_3
-; GFX11-FAKE16-NEXT: .LBB51_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-FAKE16-NEXT: .LBB51_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB51_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: .LBB51_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v8i8_to_f64_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB51_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-NEXT: s_branch .LBB51_3
+; GFX11-NEXT: .LBB51_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB51_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: .LBB51_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9122,57 +9002,56 @@ define <2 x i32> @bitcast_v4bf16_to_v2i32(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_v2i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v7 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: .LBB66_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v2i32:
@@ -9419,48 +9298,46 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_nc_u32 v8, v8, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v5 :: v_dual_add_nc_u32 v5, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v5 :: v_dual_add_nc_u32 v9, v9, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB67_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -9668,35 +9545,41 @@ define <8 x i8> @bitcast_v2i32_to_v8i8(<2 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 3, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 3, v8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: .LBB68_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2i32_to_v8i8:
@@ -9902,56 +9785,48 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB69_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB69_3
; GFX11-TRUE16-NEXT: .LBB69_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
; GFX11-TRUE16-NEXT: .LBB69_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB69_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
; GFX11-TRUE16-NEXT: .LBB69_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2i32_to_v8i8_scalar:
@@ -10204,8 +10079,8 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v2i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -10218,12 +10093,12 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -10236,11 +10111,11 @@ define <2 x i32> @bitcast_v8i8_to_v2i32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2
; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -10518,122 +10393,62 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v2i32_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
-; GFX11-TRUE16-NEXT: s_branch .LBB71_3
-; GFX11-TRUE16-NEXT: .LBB71_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-TRUE16-NEXT: .LBB71_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB71_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: .LBB71_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v2i32_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB71_3
-; GFX11-FAKE16-NEXT: .LBB71_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-FAKE16-NEXT: .LBB71_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB71_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: .LBB71_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v8i8_to_v2i32_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-NEXT: s_branch .LBB71_3
+; GFX11-NEXT: .LBB71_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB71_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: .LBB71_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11959,62 +11774,61 @@ define <2 x float> @bitcast_v4bf16_to_v2f32(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_v2f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v7 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v2f32:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: .LBB82_2: ; %end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v2f32:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-FAKE16-NEXT: v_cmpx_ne_u32_e32 0, v2
; GFX11-FAKE16-NEXT: s_xor_b32 s0, exec_lo, s0
@@ -12256,48 +12070,46 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_nc_u32 v8, v8, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v5 :: v_dual_add_nc_u32 v5, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v5 :: v_dual_add_nc_u32 v9, v9, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB83_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -12505,34 +12317,40 @@ define <8 x i8> @bitcast_v2f32_to_v8i8(<2 x float> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 1.0, v9 :: v_dual_add_f32 v8, 1.0, v8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: .LBB84_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2f32_to_v8i8:
@@ -12750,51 +12568,52 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB85_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB85_3
; GFX11-TRUE16-NEXT: .LBB85_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
; GFX11-TRUE16-NEXT: .LBB85_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB85_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, s1, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, s0, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, s0, 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: .LBB85_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB85_6
+; GFX11-TRUE16-NEXT: .LBB85_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v7, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v5, s2
+; GFX11-TRUE16-NEXT: .LBB85_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2f32_to_v8i8_scalar:
@@ -13052,8 +12871,8 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v2f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -13066,12 +12885,12 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -13084,11 +12903,11 @@ define <2 x float> @bitcast_v8i8_to_v2f32(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2
; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -13366,122 +13185,62 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB87_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v2f32_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB87_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
-; GFX11-TRUE16-NEXT: s_branch .LBB87_3
-; GFX11-TRUE16-NEXT: .LBB87_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-TRUE16-NEXT: .LBB87_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB87_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: .LBB87_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v2f32_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB87_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB87_3
-; GFX11-FAKE16-NEXT: .LBB87_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-FAKE16-NEXT: .LBB87_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB87_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: .LBB87_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v8i8_to_v2f32_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB87_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-NEXT: s_branch .LBB87_3
+; GFX11-NEXT: .LBB87_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB87_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: .LBB87_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -14423,39 +14182,49 @@ define <4 x i16> @bitcast_v4bf16_to_v4i16(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v10 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v7 :: v_dual_and_b32 v0, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v6, v7 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11-TRUE16-NEXT: .LBB94_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -14716,44 +14485,41 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, 16, s0
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s1, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB95_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -14982,35 +14748,41 @@ define <8 x i8> @bitcast_v4i16_to_v8i8(<4 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_pk_add_u16 v9, v9, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: .LBB96_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i16_to_v8i8:
@@ -15245,51 +15017,52 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB97_3
; GFX11-TRUE16-NEXT: .LBB97_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
; GFX11-TRUE16-NEXT: .LBB97_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v9, s1, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: .LBB97_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB97_6
+; GFX11-TRUE16-NEXT: .LBB97_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v7, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v5, s2
+; GFX11-TRUE16-NEXT: .LBB97_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i16_to_v8i8_scalar:
@@ -15555,8 +15328,8 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -15569,12 +15342,12 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -15587,11 +15360,11 @@ define <4 x i16> @bitcast_v8i8_to_v4i16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2
; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -15882,122 +15655,62 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4i16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
-; GFX11-TRUE16-NEXT: s_branch .LBB99_3
-; GFX11-TRUE16-NEXT: .LBB99_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-TRUE16-NEXT: .LBB99_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB99_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: .LBB99_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v4i16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB99_3
-; GFX11-FAKE16-NEXT: .LBB99_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-FAKE16-NEXT: .LBB99_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB99_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: .LBB99_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v8i8_to_v4i16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB99_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-NEXT: s_branch .LBB99_3
+; GFX11-NEXT: .LBB99_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB99_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: .LBB99_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -16446,59 +16159,55 @@ define <4 x half> @bitcast_v4bf16_to_v4f16(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_v4f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
-; GFX11-TRUE16-NEXT: .LBB102_4: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v4f16:
@@ -16758,48 +16467,48 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, 16, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s1, 16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v3, v7 :: v_dual_add_nc_u32 v8, v8, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB103_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -17037,35 +16746,41 @@ define <8 x i8> @bitcast_v4f16_to_v8i8(<4 x half> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v2
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_pk_add_f16 v9, 0x200, v9 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v8, 0x200, v8 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %end
+; GFX11-TRUE16-NEXT: .LBB104_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f16_to_v8i8:
@@ -17310,51 +17025,52 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB105_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB105_3
; GFX11-TRUE16-NEXT: .LBB105_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
; GFX11-TRUE16-NEXT: .LBB105_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v9, 0x200, s1 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v8, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: .LBB105_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB105_6
+; GFX11-TRUE16-NEXT: .LBB105_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v7, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v5, s2
+; GFX11-TRUE16-NEXT: .LBB105_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f16_to_v8i8_scalar:
@@ -17620,8 +17336,8 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -17634,12 +17350,12 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -17652,11 +17368,11 @@ define <4 x half> @bitcast_v8i8_to_v4f16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2
; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -17947,122 +17663,62 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4f16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
-; GFX11-TRUE16-NEXT: s_branch .LBB107_3
-; GFX11-TRUE16-NEXT: .LBB107_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-TRUE16-NEXT: .LBB107_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB107_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: .LBB107_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v4f16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB107_3
-; GFX11-FAKE16-NEXT: .LBB107_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-FAKE16-NEXT: .LBB107_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB107_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: .LBB107_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v8i8_to_v4f16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB107_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-NEXT: s_branch .LBB107_3
+; GFX11-NEXT: .LBB107_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB107_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: .LBB107_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -18296,80 +17952,80 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v4bf16_to_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v1.h
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v2 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v2, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v11, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.h
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v3, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v9.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v8i8:
@@ -18701,90 +18357,92 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
; GFX11-TRUE16-NEXT: s_branch .LBB109_3
; GFX11-TRUE16-NEXT: .LBB109_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
; GFX11-TRUE16-NEXT: .LBB109_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB109_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s1, 16
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s2
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v7 :: v_dual_add_nc_u32 v6, v6, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v0, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: .LBB109_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[7:8]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v8
+; GFX11-TRUE16-NEXT: s_branch .LBB109_6
+; GFX11-TRUE16-NEXT: .LBB109_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v6, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s5
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v3, s2
+; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v8i8_scalar:
@@ -19088,8 +18746,8 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
@@ -19102,12 +18760,12 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v10, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v9, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
@@ -19120,11 +18778,11 @@ define <4 x bfloat> @bitcast_v8i8_to_v4bf16(<8 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2
; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v10.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v9.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v6.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v9.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v10.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
@@ -19413,122 +19071,62 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4bf16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
-; GFX11-TRUE16-NEXT: s_branch .LBB111_3
-; GFX11-TRUE16-NEXT: .LBB111_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-TRUE16-NEXT: .LBB111_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB111_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-TRUE16-NEXT: .LBB111_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v4bf16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-FAKE16-NEXT: s_branch .LBB111_3
-; GFX11-FAKE16-NEXT: .LBB111_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-FAKE16-NEXT: .LBB111_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB111_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: .LBB111_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v8i8_to_v4bf16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB111_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-NEXT: s_branch .LBB111_3
+; GFX11-NEXT: .LBB111_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-NEXT: .LBB111_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: .LBB111_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index 2a5ccd4e7c1f6..c421dcf7835b5 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -554,49 +554,53 @@ define <12 x i8> @bitcast_v3i32_to_v12i8(<3 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v3i32_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v2 :: v_dual_mov_b32 v11, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v13, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, v1
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB4_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-TRUE16-NEXT: .LBB4_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB4_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 3, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 3, v14
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 3, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB4_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v3i32_to_v12i8:
@@ -855,47 +859,35 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB5_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s6 :: v_dual_mov_b32 v0, s0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB5_3
; GFX11-TRUE16-NEXT: .LBB5_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s14, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
; GFX11-TRUE16-NEXT: .LBB5_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s14, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
@@ -903,22 +895,26 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v9, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s1
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[0:1], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v1, s3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
; GFX11-TRUE16-NEXT: .LBB5_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s10
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v3i32_to_v12i8_scalar:
@@ -1244,9 +1240,9 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v3i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -1259,16 +1255,16 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB6_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v15, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -1284,9 +1280,9 @@ define <3 x i32> @bitcast_v12i8_to_v3i32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB6_2
; GFX11-TRUE16-NEXT: .LBB6_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v15.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
@@ -1643,141 +1639,71 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v3i32_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s23 :: v_dual_mov_b32 v6, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s21 :: v_dual_mov_b32 v9, s20
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s19 :: v_dual_mov_b32 v12, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s18 :: v_dual_mov_b32 v8, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s17 :: v_dual_mov_b32 v10, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s2 :: v_dual_mov_b32 v14, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB7_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v13, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v11, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v6, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v14, v10, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v12, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v9, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v15
-; GFX11-TRUE16-NEXT: s_branch .LBB7_3
-; GFX11-TRUE16-NEXT: .LBB7_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX11-TRUE16-NEXT: .LBB7_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v14
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 3, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 3, v6
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v10, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v11, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v9, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: .LBB7_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v3i32_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB7_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-FAKE16-NEXT: s_branch .LBB7_3
-; GFX11-FAKE16-NEXT: .LBB7_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX11-FAKE16-NEXT: .LBB7_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB7_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: .LBB7_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v12i8_to_v3i32_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB7_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-NEXT: s_branch .LBB7_3
+; GFX11-NEXT: .LBB7_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX11-NEXT: .LBB7_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: .LBB7_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2246,74 +2172,74 @@ define <3 x i32> @bitcast_v6bf16_to_v3i32(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v3i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v3
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v8 :: v_dual_and_b32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
-; GFX11-TRUE16-NEXT: .LBB10_4: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
+; GFX11-TRUE16-NEXT: .LBB10_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v3i32:
@@ -4008,48 +3934,52 @@ define <12 x i8> @bitcast_v3f32_to_v12i8(<3 x float> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v3f32_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v2 :: v_dual_mov_b32 v11, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v13, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, v1
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB20_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-TRUE16-NEXT: .LBB20_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB20_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 1.0, v12 :: v_dual_add_f32 v11, 1.0, v11
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 1.0, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[11:12]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v11
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 1.0, v8
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 1.0, v14 :: v_dual_add_f32 v13, 1.0, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB20_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v3f32_to_v12i8:
@@ -4321,47 +4251,35 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB21_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v8, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s5
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB21_3
; GFX11-TRUE16-NEXT: .LBB21_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
; GFX11-TRUE16-NEXT: .LBB21_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB21_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, s2, 1.0
@@ -4370,17 +4288,27 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: .LBB21_5: ; %end
+; GFX11-TRUE16-NEXT: s_branch .LBB21_6
+; GFX11-TRUE16-NEXT: .LBB21_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v13, s13
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB21_6: ; %end
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v12.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v3f32_to_v12i8_scalar:
@@ -4712,9 +4640,9 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v3f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -4727,16 +4655,16 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB22_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v15, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -4752,9 +4680,9 @@ define <3 x float> @bitcast_v12i8_to_v3f32(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
; GFX11-TRUE16-NEXT: .LBB22_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v15.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v3.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
@@ -5111,141 +5039,71 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v3f32_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s23 :: v_dual_mov_b32 v6, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s21 :: v_dual_mov_b32 v9, s20
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s19 :: v_dual_mov_b32 v12, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s18 :: v_dual_mov_b32 v8, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s17 :: v_dual_mov_b32 v10, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s2 :: v_dual_mov_b32 v14, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB23_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v13, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v11, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v6, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v14, v10, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v12, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v9, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v15
-; GFX11-TRUE16-NEXT: s_branch .LBB23_3
-; GFX11-TRUE16-NEXT: .LBB23_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX11-TRUE16-NEXT: .LBB23_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v14
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 3, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 3, v6
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v10, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v11, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v9, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: .LBB23_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v3f32_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB23_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-FAKE16-NEXT: s_branch .LBB23_3
-; GFX11-FAKE16-NEXT: .LBB23_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX11-FAKE16-NEXT: .LBB23_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: .LBB23_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v12i8_to_v3f32_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB23_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-NEXT: s_branch .LBB23_3
+; GFX11-NEXT: .LBB23_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX11-NEXT: .LBB23_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: .LBB23_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5725,74 +5583,74 @@ define <3 x float> @bitcast_v6bf16_to_v3f32(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v3f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v3
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v2 :: v_dual_mov_b32 v4, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v8 :: v_dual_and_b32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
-; GFX11-TRUE16-NEXT: .LBB26_4: ; %end
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
+; GFX11-TRUE16-NEXT: .LBB26_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v3f32:
@@ -7614,10 +7472,10 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -7630,17 +7488,17 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB36_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v16, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v16, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -7655,14 +7513,14 @@ define <6 x bfloat> @bitcast_v12i8_to_v6bf16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB36_2
; GFX11-TRUE16-NEXT: .LBB36_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v16.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v15.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v8.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v16.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
@@ -8031,141 +7889,71 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6bf16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s23 :: v_dual_mov_b32 v7, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s21 :: v_dual_mov_b32 v10, s20
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s19 :: v_dual_mov_b32 v13, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v9, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v11, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s2 :: v_dual_mov_b32 v15, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB37_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v12, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v15, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v13, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB37_3
-; GFX11-TRUE16-NEXT: .LBB37_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB37_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB37_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v15
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v14
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-TRUE16-NEXT: .LBB37_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v6bf16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB37_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-FAKE16-NEXT: s_branch .LBB37_3
-; GFX11-FAKE16-NEXT: .LBB37_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB37_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB37_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: .LBB37_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v12i8_to_v6bf16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB37_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-NEXT: s_branch .LBB37_3
+; GFX11-NEXT: .LBB37_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB37_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: .LBB37_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -8479,110 +8267,109 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v2 :: v_dual_mov_b32 v14, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v13, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[2:3]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[0:1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v2.h
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[15:16]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB38_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, 0x7fc07fc0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v14.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, 0x7fc07fc0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v9, v10 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v4 :: v_dual_add_f32 v4, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v1, v12 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v0, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v8.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[16:17]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v10.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[15:16]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v15
; GFX11-TRUE16-NEXT: .LBB38_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v15.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v12i8:
@@ -9034,39 +8821,27 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB39_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s5
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB39_3
; GFX11-TRUE16-NEXT: .LBB39_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
; GFX11-TRUE16-NEXT: .LBB39_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
@@ -9078,76 +8853,87 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s3
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v4
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, 0x7fc07fc0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v7
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v4, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v3, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v6.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[11:12]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: .LBB39_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v12.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[11:12]
+; GFX11-TRUE16-NEXT: s_branch .LBB39_6
+; GFX11-TRUE16-NEXT: .LBB39_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v5, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v9, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s13 :: v_dual_mov_b32 v7, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s12 :: v_dual_mov_b32 v13, s9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
+; GFX11-TRUE16-NEXT: .LBB39_6: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v12i8_scalar:
@@ -9554,10 +9340,10 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -9570,17 +9356,17 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB40_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v16, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v16, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -9595,14 +9381,14 @@ define <6 x half> @bitcast_v12i8_to_v6f16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB40_2
; GFX11-TRUE16-NEXT: .LBB40_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v16.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v15.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v8.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v16.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
@@ -9974,141 +9760,71 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6f16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s23 :: v_dual_mov_b32 v7, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s21 :: v_dual_mov_b32 v10, s20
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s19 :: v_dual_mov_b32 v13, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v9, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v11, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s2 :: v_dual_mov_b32 v15, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB41_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v12, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v15, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v13, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB41_3
-; GFX11-TRUE16-NEXT: .LBB41_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB41_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB41_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v15
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v14
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-TRUE16-NEXT: .LBB41_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v6f16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB41_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-FAKE16-NEXT: s_branch .LBB41_3
-; GFX11-FAKE16-NEXT: .LBB41_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB41_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB41_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: .LBB41_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v12i8_to_v6f16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-NEXT: s_branch .LBB41_3
+; GFX11-NEXT: .LBB41_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB41_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: .LBB41_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10326,50 +10042,56 @@ define <12 x i8> @bitcast_v6f16_to_v12i8(<6 x half> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6f16_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v2 :: v_dual_mov_b32 v13, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v16, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v15, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB42_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[12:13]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v12
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[15:16]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v15
+; GFX11-TRUE16-NEXT: .LBB42_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB42_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v16, 0x200, v16 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v15, 0x200, v15 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v13, 0x200, v13 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v12, 0x200, v12 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, 0x7e007e00
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, 0x7e007e00
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[12:13]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[15:16]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v15
; GFX11-TRUE16-NEXT: .LBB42_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v13.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6f16_to_v12i8:
@@ -10685,69 +10407,66 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB43_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s5
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB43_3
; GFX11-TRUE16-NEXT: .LBB43_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
; GFX11-TRUE16-NEXT: .LBB43_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB43_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, s2 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, 0x7e007e00
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v8, 0x200, s2 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, 0x7e007e00
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 8, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: .LBB43_5: ; %end
+; GFX11-TRUE16-NEXT: s_branch .LBB43_6
+; GFX11-TRUE16-NEXT: .LBB43_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v7, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s13 :: v_dual_mov_b32 v13, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB43_6: ; %end
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v12.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6f16_to_v12i8_scalar:
@@ -11098,10 +10817,10 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v12
@@ -11114,17 +10833,17 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB44_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v16, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v14, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v16, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
@@ -11139,14 +10858,14 @@ define <6 x i16> @bitcast_v12i8_to_v6i16(<12 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB44_2
; GFX11-TRUE16-NEXT: .LBB44_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v16.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v15.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v15.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v14.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v13.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v8.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v16.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v7.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
@@ -11518,141 +11237,71 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6i16_scalar:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s23 :: v_dual_mov_b32 v7, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s21 :: v_dual_mov_b32 v10, s20
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s19 :: v_dual_mov_b32 v13, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v9, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v11, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s2 :: v_dual_mov_b32 v15, s0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB45_2
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v12, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v15, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v16, v13, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB45_3
-; GFX11-TRUE16-NEXT: .LBB45_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-TRUE16-NEXT: .LBB45_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB45_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v15
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v14
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v12
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v6, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-TRUE16-NEXT: .LBB45_5: ; %end
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v6i16_scalar:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB45_2
-; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-FAKE16-NEXT: s_branch .LBB45_3
-; GFX11-FAKE16-NEXT: .LBB45_2:
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
-; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-FAKE16-NEXT: .LBB45_3: ; %Flow
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB45_5
-; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
-; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-FAKE16-NEXT: .LBB45_5: ; %end
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: bitcast_v12i8_to_v6i16_scalar:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-NEXT: s_mov_b32 s4, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
+; GFX11-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-NEXT: s_branch .LBB45_3
+; GFX11-NEXT: .LBB45_2:
+; GFX11-NEXT: s_mov_b32 s4, -1
+; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-NEXT: .LBB45_3: ; %Flow
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-NEXT: s_add_i32 s0, s0, 3
+; GFX11-NEXT: s_add_i32 s2, s2, 3
+; GFX11-NEXT: s_add_i32 s16, s16, 3
+; GFX11-NEXT: s_add_i32 s18, s18, 3
+; GFX11-NEXT: s_add_i32 s20, s20, 3
+; GFX11-NEXT: s_add_i32 s22, s22, 3
+; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-NEXT: .LBB45_5: ; %end
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11868,50 +11517,55 @@ define <12 x i8> @bitcast_v6i16_to_v12i8(<6 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6i16_to_v12i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v2 :: v_dual_mov_b32 v13, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v2 :: v_dual_mov_b32 v13, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, v1
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[12:13]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v12
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-TRUE16-NEXT: .LBB46_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v14, v14, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v9, v0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, v8, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v13, v13, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, v0, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v12, v12, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v13
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[14:15], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[12:13]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6i16_to_v12i8:
@@ -12211,66 +11865,65 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB47_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v9, s5
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s6 :: v_dual_mov_b32 v1, s5
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v11, s4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
; GFX11-TRUE16-NEXT: s_branch .LBB47_3
; GFX11-TRUE16-NEXT: .LBB47_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
; GFX11-TRUE16-NEXT: .LBB47_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v9, s0, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, s2, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[12:13], 24, v[3:4]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v4
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: .LBB47_5: ; %end
+; GFX11-TRUE16-NEXT: s_branch .LBB47_6
+; GFX11-TRUE16-NEXT: .LBB47_5:
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v7, s10
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s13 :: v_dual_mov_b32 v9, s9
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: .LBB47_6: ; %end
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v12.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6i16_to_v12i8_scalar:
@@ -12561,74 +12214,72 @@ define <6 x half> @bitcast_v6bf16_to_v6f16(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v6bf16_to_v6f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_vgpr4_vgpr5_vgpr6
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v3
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v3 :: v_dual_mov_b32 v5, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v3, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v7 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v8, v12 :: v_dual_add_f32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v0, v7, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v7 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v11, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.h
-; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v5, v8, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: .LBB48_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v3 :: v_dual_mov_b32 v1, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v5
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v6f16:
@@ -13679,54 +13330,70 @@ define <6 x i16> @bitcast_v6bf16_to_v6i16(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB52_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v6, v7 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v7 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v7.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
; GFX11-TRUE16-NEXT: .LBB52_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
index f03c9195edbfa..3921b453cf8e1 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-from-float-fp8-f16-hw.ll
@@ -49,9 +49,9 @@ define <2 x half> @from_fp8_v2f16(<2 x i8> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 0xff bitop3:0xec
; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -74,9 +74,9 @@ define <2 x half> @from_bf8_v2f16(<2 x i8> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v1.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 0xff bitop3:0xec
; GFX1250-TRUE16-NEXT: v_cvt_pk_f16_bf8 v0, v0.l
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
index e289b558aecdf..fc93097ac50db 100644
--- a/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/arbitrary-fp-to-float-fp8-hw.ll
@@ -600,101 +600,101 @@ define <4 x float> @v4_from_e5m3fnu(<4 x i8> %x) {
; GFX1170-NEXT: v_or_b32_e32 v10, v0, v4
; GFX1170-NEXT: v_cmp_eq_u32_e64 s1, 7, v4
; GFX1170-NEXT: v_sub_nc_u32_e32 v11, 31, v8
-; GFX1170-NEXT: v_add_nc_u32_e32 v17, -8, v8
+; GFX1170-NEXT: v_add_nc_u32_e32 v16, -8, v8
; GFX1170-NEXT: v_cmp_eq_u32_e64 s0, 0, v0
; GFX1170-NEXT: v_cmp_eq_u32_e64 s2, 31, v0
; GFX1170-NEXT: v_sub_nc_u32_e32 v8, 0x8d, v8
; GFX1170-NEXT: v_lshlrev_b32_e64 v11, v11, 1
; GFX1170-NEXT: v_lshl_or_b32 v0, v0, 23, v9
; GFX1170-NEXT: s_and_b32 vcc_lo, s0, vcc_lo
-; GFX1170-NEXT: v_clz_i32_u32_e32 v15, v6
; GFX1170-NEXT: s_and_b32 s0, s2, s1
+; GFX1170-NEXT: v_cmp_ne_u32_e64 s1, 0, v6
; GFX1170-NEXT: v_xor_b32_e32 v4, v4, v11
; GFX1170-NEXT: v_add_nc_u32_e32 v0, 0x38000000, v0
-; GFX1170-NEXT: v_cmp_ne_u32_e64 s1, 0, v6
+; GFX1170-NEXT: v_clz_i32_u32_e32 v11, v6
; GFX1170-NEXT: v_cmp_eq_u32_e64 s2, 0, v2
; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1170-NEXT: v_lshlrev_b32_e32 v4, v17, v4
+; GFX1170-NEXT: v_lshlrev_b32_e32 v4, v16, v4
; GFX1170-NEXT: s_and_b32 s1, s2, s1
; GFX1170-NEXT: v_cmp_eq_u32_e64 s2, 31, v2
; GFX1170-NEXT: v_lshl_or_b32 v4, v8, 23, v4
; GFX1170-NEXT: v_and_b32_e32 v5, 7, v1
; GFX1170-NEXT: v_bfe_u32 v1, v1, 3, 5
-; GFX1170-NEXT: v_sub_nc_u32_e32 v8, 31, v15
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1170-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
; GFX1170-NEXT: v_clz_i32_u32_e32 v12, v5
-; GFX1170-NEXT: v_lshlrev_b32_e32 v13, 20, v5
-; GFX1170-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v10
-; GFX1170-NEXT: v_and_b32_e32 v7, 7, v3
; GFX1170-NEXT: v_cmp_ne_u32_e64 s3, 0, v5
-; GFX1170-NEXT: v_sub_nc_u32_e32 v16, 31, v12
-; GFX1170-NEXT: v_add_nc_u32_e32 v9, -8, v12
-; GFX1170-NEXT: v_sub_nc_u32_e32 v12, 0x8d, v12
-; GFX1170-NEXT: v_lshl_or_b32 v13, v1, 23, v13
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1170-NEXT: v_cmp_eq_u32_e64 s4, 0, v1
-; GFX1170-NEXT: v_lshlrev_b32_e64 v16, v16, 1
-; GFX1170-NEXT: v_or_b32_e32 v14, v1, v5
-; GFX1170-NEXT: v_lshlrev_b32_e64 v4, v8, 1
-; GFX1170-NEXT: v_add_nc_u32_e32 v8, 0x38000000, v13
+; GFX1170-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v10
+; GFX1170-NEXT: v_and_b32_e32 v7, 7, v3
+; GFX1170-NEXT: v_sub_nc_u32_e32 v15, 31, v12
+; GFX1170-NEXT: v_add_nc_u32_e32 v16, -8, v12
+; GFX1170-NEXT: v_sub_nc_u32_e32 v8, 0x8d, v12
; GFX1170-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc_lo
-; GFX1170-NEXT: v_xor_b32_e32 v16, v5, v16
-; GFX1170-NEXT: v_clz_i32_u32_e32 v11, v7
; GFX1170-NEXT: s_and_b32 vcc_lo, s4, s3
-; GFX1170-NEXT: v_xor_b32_e32 v4, v6, v4
+; GFX1170-NEXT: v_lshlrev_b32_e64 v15, v15, 1
+; GFX1170-NEXT: v_lshlrev_b32_e32 v13, 20, v5
+; GFX1170-NEXT: v_sub_nc_u32_e32 v4, 31, v11
+; GFX1170-NEXT: v_or_b32_e32 v14, v1, v5
+; GFX1170-NEXT: v_clz_i32_u32_e32 v9, v7
+; GFX1170-NEXT: v_xor_b32_e32 v15, v5, v15
+; GFX1170-NEXT: v_lshl_or_b32 v12, v1, 23, v13
+; GFX1170-NEXT: v_lshlrev_b32_e64 v4, v4, 1
+; GFX1170-NEXT: v_add_nc_u32_e32 v10, -8, v11
; GFX1170-NEXT: v_cndmask_b32_e64 v0, v0, 0x7fc00000, s0
-; GFX1170-NEXT: v_lshlrev_b32_e32 v9, v9, v16
-; GFX1170-NEXT: v_sub_nc_u32_e32 v10, 31, v11
+; GFX1170-NEXT: v_lshlrev_b32_e32 v13, v16, v15
+; GFX1170-NEXT: v_add_nc_u32_e32 v12, 0x38000000, v12
+; GFX1170-NEXT: v_xor_b32_e32 v4, v6, v4
; GFX1170-NEXT: v_cmp_eq_u32_e64 s0, 31, v1
; GFX1170-NEXT: v_bfe_u32 v3, v3, 3, 5
+; GFX1170-NEXT: v_lshl_or_b32 v8, v8, 23, v13
; GFX1170-NEXT: v_cmp_ne_u32_e64 s3, 0, v7
-; GFX1170-NEXT: v_lshl_or_b32 v9, v12, 23, v9
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1170-NEXT: v_cmp_eq_u32_e64 s4, 0, v3
-; GFX1170-NEXT: v_dual_cndmask_b32 v8, v8, v9 :: v_dual_add_nc_u32 v9, -8, v15
+; GFX1170-NEXT: v_lshlrev_b32_e32 v4, v10, v4
+; GFX1170-NEXT: v_sub_nc_u32_e32 v10, 0x8d, v11
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX1170-NEXT: v_dual_cndmask_b32 v8, v12, v8 :: v_dual_lshlrev_b32 v11, 20, v6
+; GFX1170-NEXT: v_sub_nc_u32_e32 v12, 31, v9
; GFX1170-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_lshlrev_b32_e32 v4, v9, v4
+; GFX1170-NEXT: v_lshl_or_b32 v4, v10, 23, v4
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1170-NEXT: v_lshl_or_b32 v10, v2, 23, v11
+; GFX1170-NEXT: v_lshlrev_b32_e32 v11, 20, v7
+; GFX1170-NEXT: v_cmp_eq_u32_e64 s4, 0, v3
; GFX1170-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc_lo
; GFX1170-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v5
-; GFX1170-NEXT: v_lshlrev_b32_e64 v5, v10, 1
-; GFX1170-NEXT: v_sub_nc_u32_e32 v9, 0x8d, v15
-; GFX1170-NEXT: v_lshlrev_b32_e32 v10, 20, v6
+; GFX1170-NEXT: v_lshlrev_b32_e64 v5, v12, 1
+; GFX1170-NEXT: v_add_nc_u32_e32 v10, 0x38000000, v10
; GFX1170-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1170-NEXT: v_xor_b32_e32 v1, v7, v5
-; GFX1170-NEXT: v_add_nc_u32_e32 v5, -8, v11
-; GFX1170-NEXT: v_lshl_or_b32 v4, v9, 23, v4
-; GFX1170-NEXT: v_lshl_or_b32 v9, v2, 23, v10
-; GFX1170-NEXT: v_lshlrev_b32_e32 v10, 20, v7
-; GFX1170-NEXT: v_lshlrev_b32_e32 v1, v5, v1
-; GFX1170-NEXT: v_sub_nc_u32_e32 v5, 0x8d, v11
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1170-NEXT: v_add_nc_u32_e32 v9, 0x38000000, v9
-; GFX1170-NEXT: v_lshl_or_b32 v10, v3, 23, v10
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_lshl_or_b32 v1, v5, 23, v1
-; GFX1170-NEXT: v_cndmask_b32_e64 v4, v9, v4, s1
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1170-NEXT: v_add_nc_u32_e32 v5, 0x38000000, v10
-; GFX1170-NEXT: v_or_b32_e32 v10, v2, v6
+; GFX1170-NEXT: v_add_nc_u32_e32 v5, -8, v9
+; GFX1170-NEXT: v_cndmask_b32_e64 v4, v10, v4, s1
; GFX1170-NEXT: s_and_b32 s1, s4, s3
; GFX1170-NEXT: v_cmp_eq_u32_e64 s4, 31, v3
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_lshlrev_b32_e32 v1, v5, v1
+; GFX1170-NEXT: v_sub_nc_u32_e32 v5, 0x8d, v9
+; GFX1170-NEXT: v_lshl_or_b32 v9, v3, 23, v11
+; GFX1170-NEXT: v_lshl_or_b32 v1, v5, 23, v1
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_add_nc_u32_e32 v5, 0x38000000, v9
+; GFX1170-NEXT: v_or_b32_e32 v9, v2, v6
; GFX1170-NEXT: v_cndmask_b32_e64 v1, v5, v1, s1
; GFX1170-NEXT: v_or_b32_e32 v5, v3, v7
-; GFX1170-NEXT: v_cmp_ne_u32_e64 s1, 0, v10
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1170-NEXT: v_cmp_ne_u32_e64 s1, 0, v9
; GFX1170-NEXT: v_cmp_ne_u32_e64 s3, 0, v5
+; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1170-NEXT: v_cndmask_b32_e64 v4, 0, v4, s1
; GFX1170-NEXT: v_cmp_eq_u32_e64 s1, 7, v6
-; GFX1170-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(SALU_CYCLE_1)
; GFX1170-NEXT: v_cndmask_b32_e64 v5, 0, v1, s3
; GFX1170-NEXT: v_cmp_eq_u32_e64 s3, 7, v7
; GFX1170-NEXT: v_cndmask_b32_e64 v1, v8, 0x7fc00000, s0
; GFX1170-NEXT: s_and_b32 s0, s2, s1
+; GFX1170-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1170-NEXT: v_cndmask_b32_e64 v2, v4, 0x7fc00000, s0
; GFX1170-NEXT: s_and_b32 s0, s4, s3
-; GFX1170-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1170-NEXT: v_cndmask_b32_e64 v3, v5, 0x7fc00000, s0
; GFX1170-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 274df427de416..04f1b409b416b 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -2781,33 +2781,18 @@ define bfloat @test_byval(ptr addrspace(5) byval(bfloat) %bv, bfloat %val) #0 {
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], s32
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: test_byval:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11TRUE16-NEXT: scratch_store_b16 off, v1, s32
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: test_byval:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: scratch_store_b16 off, v0, s32
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: test_byval:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX1250TRUE16-NEXT: scratch_store_b16 off, v1, s32
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: test_byval:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: scratch_store_b16 off, v0, s32
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: test_byval:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: scratch_store_b16 off, v0, s32
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: test_byval:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: scratch_store_b16 off, v0, s32
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
store bfloat %val, ptr addrspace(5) %bv
%retval = load bfloat, ptr addrspace(5) %bv
ret bfloat %retval
@@ -5369,93 +5354,48 @@ define { <32 x i32>, bfloat } @test_overflow_stack(bfloat %a, <32 x i32> %b) #0
; GFX10-NEXT: buffer_store_short v1, v0, s[0:3], 0 offen offset:128
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: test_overflow_stack:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: s_clause 0x2
-; GFX11TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX11TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11TRUE16-NEXT: s_clause 0x3
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
-; GFX11TRUE16-NEXT: s_clause 0x1
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[2:5], off
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: s_clause 0x2
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
-; GFX11TRUE16-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
-; GFX11TRUE16-NEXT: scratch_store_b16 v0, v1, off offset:128
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: test_overflow_stack:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: s_clause 0x2
-; GFX11FAKE16-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX11FAKE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX11FAKE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11FAKE16-NEXT: s_clause 0x5
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[2:5], off
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11FAKE16-NEXT: s_clause 0x2
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
-; GFX11FAKE16-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
-; GFX11FAKE16-NEXT: scratch_store_b16 v0, v1, off offset:128
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX1250TRUE16-LABEL: test_overflow_stack:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: s_clause 0x2
-; GFX1250TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX1250TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX1250TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX1250TRUE16-NEXT: s_clause 0x3
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
-; GFX1250TRUE16-NEXT: s_clause 0x1
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[2:5], off
-; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250TRUE16-NEXT: s_clause 0x2
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
-; GFX1250TRUE16-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
-; GFX1250TRUE16-NEXT: scratch_store_b16 v0, v1, off offset:128
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX11-LABEL: test_overflow_stack:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:8
+; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4
+; GFX11-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-NEXT: s_clause 0x5
+; GFX11-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
+; GFX11-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
+; GFX11-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
+; GFX11-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
+; GFX11-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
+; GFX11-NEXT: scratch_store_b128 v0, v[2:5], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_clause 0x2
+; GFX11-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
+; GFX11-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
+; GFX11-NEXT: scratch_store_b16 v0, v1, off offset:128
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250FAKE16-LABEL: test_overflow_stack:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: s_clause 0x2
-; GFX1250FAKE16-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX1250FAKE16-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX1250FAKE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX1250FAKE16-NEXT: s_clause 0x5
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[2:5], off
-; GFX1250FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250FAKE16-NEXT: s_clause 0x2
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
-; GFX1250FAKE16-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
-; GFX1250FAKE16-NEXT: scratch_store_b16 v0, v1, off offset:128
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: test_overflow_stack:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_clause 0x2
+; GFX1250-NEXT: scratch_load_b32 v33, off, s32 offset:8
+; GFX1250-NEXT: scratch_load_b32 v32, off, s32 offset:4
+; GFX1250-NEXT: scratch_load_b32 v31, off, s32
+; GFX1250-NEXT: s_clause 0x5
+; GFX1250-NEXT: scratch_store_b128 v0, v[22:25], off offset:80
+; GFX1250-NEXT: scratch_store_b128 v0, v[18:21], off offset:64
+; GFX1250-NEXT: scratch_store_b128 v0, v[14:17], off offset:48
+; GFX1250-NEXT: scratch_store_b128 v0, v[10:13], off offset:32
+; GFX1250-NEXT: scratch_store_b128 v0, v[6:9], off offset:16
+; GFX1250-NEXT: scratch_store_b128 v0, v[2:5], off
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: s_clause 0x2
+; GFX1250-NEXT: scratch_store_b128 v0, v[30:33], off offset:112
+; GFX1250-NEXT: scratch_store_b128 v0, v[26:29], off offset:96
+; GFX1250-NEXT: scratch_store_b16 v0, v1, off offset:128
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%ins.0 = insertvalue { <32 x i32>, bfloat } poison, <32 x i32> %b, 0
%ins.1 = insertvalue { <32 x i32>, bfloat } %ins.0 ,bfloat %a, 1
ret { <32 x i32>, bfloat } %ins.1
@@ -9205,39 +9145,22 @@ define bfloat @v_fadd_bf16(bfloat %a, bfloat %b) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fadd_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fadd_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_fadd_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_fadd_bf16:
; GFX1250: ; %bb.0:
@@ -9362,10 +9285,10 @@ define <2 x bfloat> @v_fadd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_fadd_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_add_f32_e32 v2, v3, v2
@@ -9377,12 +9300,14 @@ define <2 x bfloat> @v_fadd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v2bf16:
@@ -9575,36 +9500,38 @@ define <3 x bfloat> @v_fadd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_fadd_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_add_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v3bf16:
@@ -9842,41 +9769,49 @@ define <4 x bfloat> @v_fadd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_fadd_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
; GFX11TRUE16-NEXT: v_dual_add_f32 v3, v7, v6 :: v_dual_add_f32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v4bf16:
@@ -10280,77 +10215,86 @@ define <8 x bfloat> @v_fadd_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v7, 16, v7
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_add_f32_e32 v7, v3, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v9, v10, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v3, v3, v7
+; GFX11TRUE16-NEXT: v_add_f32_e32 v7, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v8
; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v8, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_add_f32_e32 v6, v2, v6
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v11, v7, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v11, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v6, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v7, v12, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v7, v9 :: v_dual_and_b32 v9, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.l
+; GFX11TRUE16-NEXT: v_dual_add_f32 v7, v11, v10 :: v_dual_and_b32 v10, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v5, v12, v11
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11TRUE16-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v9, v14, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11TRUE16-NEXT: v_add_f32_e32 v5, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v11, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v10, v11 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v12, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v8bf16:
@@ -11103,152 +11047,175 @@ define <16 x bfloat> @v_fadd_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v19, v18 :: v_dual_and_b32 v21, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_add_f32_e32 v16, v17, v16
-; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v7, v7, v15
+; GFX11TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v14
+; GFX11TRUE16-NEXT: v_dual_add_f32 v15, v19, v15 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v6, v6, v14
+; GFX11TRUE16-NEXT: v_add_f32_e32 v16, v17, v16
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v16
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v14, v6, v14 :: v_dual_add_f32 v15, v7, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v17, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v16
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v16, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v17, v18, vcc_lo
; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v15, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v19, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v20, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v15
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v16.l
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v6
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v6, v17, v18 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_add_f32_e32 v14, v19, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v14
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v16, v20 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v15.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v14, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v17, v18 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v5, v5, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v12
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v14.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v12, v4, v12 :: v_dual_add_f32 v13, v5, v13
-; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v19, v18 :: v_dual_add_f32 v17, v22, v21
-; GFX11TRUE16-NEXT: v_bfe_u32 v22, v12, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v13, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v20, v18, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v21, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v16 :: v_dual_and_b32 v20, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v19, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v5, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v19, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_dual_add_f32 v13, v20, v13 :: v_dual_and_b32 v20, 0xffff0000, v11
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v13
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v22, v12, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_add_f32_e32 v3, v3, v11
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v14.l
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v13, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v13, v17, v22 :: v_dual_add_f32 v4, v4, v12
+; GFX11TRUE16-NEXT: v_dual_add_f32 v12, v21, v20 :: v_dual_and_b32 v21, 0xffff0000, v10
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v17, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_add_f32_e32 v12, v22, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v3, 16, 1
+; GFX11TRUE16-NEXT: v_add_f32_e32 v2, v2, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v17, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v3
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v2, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v17, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v12, v20, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v13.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v11.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v12, v17 :: v_dual_and_b32 v17, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v10.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v12, v19, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v16, v20, v19
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v21, v20 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v9
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v12, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v9
-; GFX11TRUE16-NEXT: v_dual_add_f32 v10, v2, v10 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v11, v3, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v16
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v11, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v20, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v19, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v10, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v2, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v10, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v17, v19 :: v_dual_lshlrev_b32 v19, 16, v8
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v11.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v9, v20, v19
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v18, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
-; GFX11TRUE16-NEXT: v_dual_add_f32 v17, v22, v21 :: v_dual_add_f32 v0, v0, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX11TRUE16-NEXT: v_add_f32_e32 v9, v18, v17
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v9, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v18, v21, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v0, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v20, v21, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v19, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v9, v18, v19 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v8
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v20, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v17, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v16bf16:
@@ -12709,274 +12676,317 @@ define <32 x bfloat> @v_fadd_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32
-; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
-; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
-; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v20, v4, v20 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_add_f32_e32 v5, v5, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
; GFX11TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v17
; GFX11TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_bfe_u32 v103, v5, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v16
+; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
+; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_dual_add_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v3, v3, v19
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
+; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
+; GFX11TRUE16-NEXT: v_bfe_u32 v119, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_add_f32 v19, v82, v81 :: v_dual_lshlrev_b32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v129, v19, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v19
+; GFX11TRUE16-NEXT: v_add_f32_e32 v2, v2, v18
+; GFX11TRUE16-NEXT: v_dual_add_f32 v18, v84, v83 :: v_dual_add_f32 v9, v9, v25
+; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v131, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add_f32_e32 v17, v86, v85
+; GFX11TRUE16-NEXT: v_dual_add_f32 v8, v8, v24 :: v_dual_and_b32 v39, 0xffff0000, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v133, v18, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
+; GFX11TRUE16-NEXT: v_bfe_u32 v83, v8, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v8
+; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX11TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11TRUE16-NEXT: v_bfe_u32 v115, v20, 16, 1
; GFX11TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; GFX11TRUE16-NEXT: v_dual_add_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
-; GFX11TRUE16-NEXT: v_dual_add_f32 v23, v7, v23 :: v_dual_and_b32 v82, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_add_f32_e32 v22, v6, v22
-; GFX11TRUE16-NEXT: v_add_f32_e32 v17, v86, v85
-; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v20, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
-; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v27
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11TRUE16-NEXT: v_bfe_u32 v87, v23, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v99, v22, 16, 1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
-; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v23
-; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v22
-; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v23, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; GFX11TRUE16-NEXT: v_dual_add_f32 v21, v5, v21 :: v_dual_lshlrev_b32 v26, 16, v26
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v19, v3, v19 :: v_dual_and_b32 v52, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_add_f32 v27, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; GFX11TRUE16-NEXT: v_dual_add_f32 v26, v10, v26 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11TRUE16-NEXT: v_dual_add_f32 v24, v8, v24 :: v_dual_and_b32 v35, 0xffff0000, v29
-; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
+; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v16
+; GFX11TRUE16-NEXT: v_dual_add_f32 v24, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
+; GFX11TRUE16-NEXT: v_add_f32_e32 v7, v7, v23
+; GFX11TRUE16-NEXT: v_dual_add_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v85, v24, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v97, v23, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v23
+; GFX11TRUE16-NEXT: v_bfe_u32 v87, v7, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_add_f32_e32 v4, v4, v20
+; GFX11TRUE16-NEXT: v_add_f32_e32 v20, v80, v71
+; GFX11TRUE16-NEXT: v_bfe_u32 v71, v9, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v9
+; GFX11TRUE16-NEXT: v_dual_add_f32 v21, v70, v69 :: v_dual_lshlrev_b32 v10, 16, v10
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_dual_add_f32 v10, v10, v26 :: v_dual_lshlrev_b32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_dual_add_f32 v26, v52, v51 :: v_dual_add_f32 v25, v54, v53
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v6, v6, v22
+; GFX11TRUE16-NEXT: v_dual_add_f32 v11, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
; GFX11TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_bfe_u32 v103, v21, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v119, v19, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v29, v13, v29 :: v_dual_and_b32 v38, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_dual_add_f32 v33, v34, v33 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11TRUE16-NEXT: v_add_f32_e32 v34, v36, v35
-; GFX11TRUE16-NEXT: v_add_f32_e32 v36, v48, v39
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v16
-; GFX11TRUE16-NEXT: v_dual_add_f32 v48, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
+; GFX11TRUE16-NEXT: v_add_f32_e32 v22, v68, v67
+; GFX11TRUE16-NEXT: v_dual_add_f32 v27, v50, v49 :: v_dual_and_b32 v38, 0xffff0000, v12
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v21
-; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v19
-; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v21, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v19, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11TRUE16-NEXT: v_add_f32_e32 v35, v38, v37
-; GFX11TRUE16-NEXT: v_dual_add_f32 v30, v14, v30 :: v_dual_and_b32 v31, 0xffff0000, v15
+; GFX11TRUE16-NEXT: v_dual_add_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_add_f32 v29, v38, v37 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
+; GFX11TRUE16-NEXT: v_dual_add_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v14, v14, v30
+; GFX11TRUE16-NEXT: v_add_f32_e32 v28, v48, v39
+; GFX11TRUE16-NEXT: v_dual_add_f32 v30, v36, v35 :: v_dual_add_f32 v33, v34, v33
+; GFX11TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v35, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v14
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v18, v2, v18
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v33, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v33
-; GFX11TRUE16-NEXT: v_bfe_u32 v14, v27, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v30, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v33
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v33, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v30
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v34, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v30, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v150, v14, v27, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v2, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v13
+; GFX11TRUE16-NEXT: v_bfe_u32 v49, v29, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v29
+; GFX11TRUE16-NEXT: v_bfe_u32 v51, v12, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11TRUE16-NEXT: v_dual_add_f32 v39, v54, v53 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_dual_add_f32 v54, v84, v83 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v34
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v29, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v36, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v34, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v4, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11TRUE16-NEXT: v_add_f32_e32 v25, v9, v25
-; GFX11TRUE16-NEXT: v_dual_add_f32 v28, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v29
-; GFX11TRUE16-NEXT: v_bfe_u32 v10, v35, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v29, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v149, v13, v36, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v53, v28, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v28
+; GFX11TRUE16-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11TRUE16-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v11
+; GFX11TRUE16-NEXT: v_bfe_u32 v65, v27, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v35
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v28, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v35, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v28
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v8, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11TRUE16-NEXT: v_add3_u32 v148, v12, v28, 0x7fff
-; GFX11TRUE16-NEXT: v_add_f32_e32 v37, v50, v49
-; GFX11TRUE16-NEXT: v_or_b32_e32 v55, 0x400000, v36
-; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v27
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v10
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_bfe_u32 v69, v26, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v26
+; GFX11TRUE16-NEXT: v_bfe_u32 v81, v25, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11TRUE16-NEXT: v_add_f32_e32 v49, v66, v65
-; GFX11TRUE16-NEXT: v_bfe_u32 v65, v37, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v37
-; GFX11TRUE16-NEXT: v_add_f32_e32 v38, v52, v51
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v148, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v37, 0x7fff
-; GFX11TRUE16-NEXT: v_add_f32_e32 v51, v70, v69
-; GFX11TRUE16-NEXT: v_bfe_u32 v69, v38, 16, 1
-; GFX11TRUE16-NEXT: v_add_f32_e32 v52, v80, v71
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v149, v55, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v25
+; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v99, v6, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v101, v22, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v22
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v38
-; GFX11TRUE16-NEXT: v_bfe_u32 v71, v25, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v38, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v25
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v150, v64, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v25, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v83, v24, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v85, v48, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v65, v66, vcc_lo
-; GFX11TRUE16-NEXT: v_add_f32_e32 v50, v68, v67
-; GFX11TRUE16-NEXT: v_bfe_u32 v67, v26, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v26
+; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v113, v21, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v115, v4, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v117, v20, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v48
-; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v26, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v48, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v97, v49, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v49
-; GFX11TRUE16-NEXT: v_bfe_u32 v101, v50, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v67, v68, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v49, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v50
-; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v50, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v113, v51, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v69, v70, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v51
-; GFX11TRUE16-NEXT: v_bfe_u32 v117, v52, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v51, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v52
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v71, v80, vcc_lo
-; GFX11TRUE16-NEXT: v_add_f32_e32 v53, v82, v81
-; GFX11TRUE16-NEXT: v_bfe_u32 v81, v39, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v39
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v52, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v129, v53, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v39, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v53
-; GFX11TRUE16-NEXT: v_bfe_u32 v131, v18, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v53, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v81, v82, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v147, v0, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v83, v84, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v133, v54, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; GFX11TRUE16-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v85, v86, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.h, v25.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.h, v26.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.h, v27.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v54
-; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v54, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v87, v96, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v16.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v24.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v97, v98, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.h, v28.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.h, v29.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.h, v30.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v24.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.h, v16.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v25.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v26.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v27.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v29.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v99, v100, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v30.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v101, v102, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v23.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v103, v112, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v21.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v113, v114, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v22.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v115, v116, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v117, v118, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v21.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v119, v128, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v20.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v131, v132, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v18.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v135, v144, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v147, v33, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v133, v134, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v19.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v28.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v145, v146, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v33.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v18.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v32
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_add_f32 v15, v15, v17 :: v_dual_and_b32 v18, 0xffff0000, v32
-; GFX11TRUE16-NEXT: v_add_f32_e32 v17, v31, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v15
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v15, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v32
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v17.l
+; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v32
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v15, v15, v18
+; GFX11TRUE16-NEXT: v_add_f32_e32 v17, v31, v17
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v15
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v17
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v15, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.h, v17.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fadd_v32bf16:
@@ -13351,37 +13361,21 @@ define bfloat @v_fadd_bf16_fpimm_0(bfloat %arg0) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fadd_bf16_fpimm_0:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fadd_bf16_fpimm_0:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add_f32_e32 v0, 1.0, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_fadd_bf16_fpimm_0:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v0, 1.0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_fadd_bf16_fpimm_0:
; GFX1250: ; %bb.0:
@@ -13459,37 +13453,21 @@ define bfloat @v_fadd_bf16_fpimm_1(bfloat %arg0) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fadd_bf16_fpimm_1:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, 0x42280000, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fadd_bf16_fpimm_1:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add_f32_e32 v0, 0x42280000, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_fadd_bf16_fpimm_1:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v0, 0x42280000, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_fadd_bf16_fpimm_1:
; GFX1250: ; %bb.0:
@@ -13573,39 +13551,22 @@ define bfloat @v_fsub_bf16(bfloat %a, bfloat %b) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fsub_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fsub_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_fsub_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_fsub_bf16:
; GFX1250: ; %bb.0:
@@ -13730,10 +13691,10 @@ define <2 x bfloat> @v_fsub_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_fsub_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_sub_f32_e32 v2, v3, v2
@@ -13745,12 +13706,14 @@ define <2 x bfloat> @v_fsub_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fsub_v2bf16:
@@ -13943,36 +13906,38 @@ define <3 x bfloat> @v_fsub_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_fsub_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_sub_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_sub_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fsub_v3bf16:
@@ -14210,41 +14175,49 @@ define <4 x bfloat> @v_fsub_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_fsub_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_sub_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v3
; GFX11TRUE16-NEXT: v_dual_sub_f32 v3, v7, v6 :: v_dual_sub_f32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fsub_v4bf16:
@@ -14373,39 +14346,22 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fmul_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fmul_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_fmul_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_fmul_bf16:
; GFX1250: ; %bb.0:
@@ -14530,10 +14486,10 @@ define <2 x bfloat> @v_fmul_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_fmul_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_mul_f32_e32 v2, v3, v2
@@ -14545,12 +14501,14 @@ define <2 x bfloat> @v_fmul_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v2bf16:
@@ -14743,36 +14701,38 @@ define <3 x bfloat> @v_fmul_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_fmul_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_mul_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v3bf16:
@@ -15010,41 +14970,49 @@ define <4 x bfloat> @v_fmul_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_fmul_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, v1, v3
; GFX11TRUE16-NEXT: v_dual_mul_f32 v3, v7, v6 :: v_dual_mul_f32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v4bf16:
@@ -15448,77 +15416,86 @@ define <8 x bfloat> @v_fmul_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v7, 16, v7
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v7, v3, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v9, v10, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v2, v2, v6 :: v_dual_mul_f32 v3, v3, v7
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v7, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v8
; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v8, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v6, v2, v6
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v11, v7, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v11, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v6, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v7, v12, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v7, v9 :: v_dual_and_b32 v9, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.l
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v7, v11, v10 :: v_dual_and_b32 v10, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, v1, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v5, v12, v11
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v0, v0, v4 :: v_dual_mul_f32 v9, v14, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v5, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v11, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v10, v11 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v12, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v8bf16:
@@ -16271,152 +16248,175 @@ define <16 x bfloat> @v_fmul_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v19, v18 :: v_dual_and_b32 v21, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v16, v17, v16
-; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v7, v7, v15
+; GFX11TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v14
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v15, v19, v15 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v6, v6, v14
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v16, v17, v16
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v16
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v14, v6, v14 :: v_dual_mul_f32 v15, v7, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v17, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v16
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v16, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v17, v18, vcc_lo
; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v15, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v19, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v20, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v15
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v16.l
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v6
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v6, v17, v18 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v14, v19, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v14
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v16, v20 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v15.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v14, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v17, v18 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v5, v5, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v12
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v14.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v12, v4, v12 :: v_dual_mul_f32 v13, v5, v13
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v19, v18 :: v_dual_mul_f32 v17, v22, v21
-; GFX11TRUE16-NEXT: v_bfe_u32 v22, v12, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v13, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v20, v18, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v21, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v16 :: v_dual_and_b32 v20, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v19, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v5, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v19, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v13, v20, v13 :: v_dual_and_b32 v20, 0xffff0000, v11
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v13
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v22, v12, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v3, v3, v11
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v14.l
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v13, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v13, v17, v22 :: v_dual_mul_f32 v4, v4, v12
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v12, v21, v20 :: v_dual_and_b32 v21, 0xffff0000, v10
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v17, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v12, v22, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v3, 16, 1
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v2, v2, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v17, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v3
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v2, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v17, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v12, v20, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v13.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v11.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v12, v17 :: v_dual_and_b32 v17, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v10.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v12, v19, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v16, v20, v19
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v21, v20 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v9
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v12, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, v1, v9
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v10, v2, v10 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v11, v3, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v16
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v11, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v20, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v19, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v10, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v2, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v10, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v17, v19 :: v_dual_lshlrev_b32 v19, 16, v8
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v11.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v9, v20, v19
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v18, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v17, v22, v21 :: v_dual_mul_f32 v0, v0, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v9, v18, v17
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v9, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v18, v21, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v0, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v20, v21, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v19, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v9, v18, v19 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v8
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v20, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v17, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v16bf16:
@@ -17877,274 +17877,317 @@ define <32 x bfloat> @v_fmul_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32
-; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
-; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
-; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v20, v4, v20 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v5, v5, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
; GFX11TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v17
; GFX11TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_bfe_u32 v103, v5, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v16
+; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
+; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v3, v3, v19
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
+; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
+; GFX11TRUE16-NEXT: v_bfe_u32 v119, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v19, v82, v81 :: v_dual_lshlrev_b32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v129, v19, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v19
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v2, v2, v18
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v18, v84, v83 :: v_dual_mul_f32 v9, v9, v25
+; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v131, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v2
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v17, v86, v85
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v8, v8, v24 :: v_dual_and_b32 v39, 0xffff0000, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v133, v18, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
+; GFX11TRUE16-NEXT: v_bfe_u32 v83, v8, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v8
+; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX11TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11TRUE16-NEXT: v_bfe_u32 v115, v20, 16, 1
; GFX11TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v23, v7, v23 :: v_dual_and_b32 v82, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v22, v6, v22
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v17, v86, v85
-; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v20, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
-; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v27
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11TRUE16-NEXT: v_bfe_u32 v87, v23, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v99, v22, 16, 1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
-; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v23
-; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v22
-; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v23, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v21, v5, v21 :: v_dual_lshlrev_b32 v26, 16, v26
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v19, v3, v19 :: v_dual_and_b32 v52, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v27, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v26, v10, v26 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v24, v8, v24 :: v_dual_and_b32 v35, 0xffff0000, v29
-; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
+; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v16
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v24, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v7, v7, v23
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v85, v24, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v97, v23, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v23
+; GFX11TRUE16-NEXT: v_bfe_u32 v87, v7, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v4, v4, v20
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v20, v80, v71
+; GFX11TRUE16-NEXT: v_bfe_u32 v71, v9, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v9
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v21, v70, v69 :: v_dual_lshlrev_b32 v10, 16, v10
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v10, v10, v26 :: v_dual_lshlrev_b32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v26, v52, v51 :: v_dual_mul_f32 v25, v54, v53
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v6, v6, v22
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v11, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
; GFX11TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_bfe_u32 v103, v21, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v119, v19, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v29, v13, v29 :: v_dual_and_b32 v38, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v33, v34, v33 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v34, v36, v35
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v36, v48, v39
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v0, v0, v16
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v48, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v22, v68, v67
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v27, v50, v49 :: v_dual_and_b32 v38, 0xffff0000, v12
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v21
-; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v19
-; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v21, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v19, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v35, v38, v37
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v30, v14, v30 :: v_dual_and_b32 v31, 0xffff0000, v15
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v29, v38, v37 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v14, v14, v30
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v28, v48, v39
+; GFX11TRUE16-NEXT: v_dual_mul_f32 v30, v36, v35 :: v_dual_mul_f32 v33, v34, v33
+; GFX11TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v35, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v14
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v18, v2, v18
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v33, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v33
-; GFX11TRUE16-NEXT: v_bfe_u32 v14, v27, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v30, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v33
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v33, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v30
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v34, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v30, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v150, v14, v27, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v2, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v13
+; GFX11TRUE16-NEXT: v_bfe_u32 v49, v29, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v29
+; GFX11TRUE16-NEXT: v_bfe_u32 v51, v12, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v39, v54, v53 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v54, v84, v83 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v34
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v29, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v36, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v34, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v4, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v25, v9, v25
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v28, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v29
-; GFX11TRUE16-NEXT: v_bfe_u32 v10, v35, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v29, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v149, v13, v36, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v53, v28, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v28
+; GFX11TRUE16-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11TRUE16-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v11
+; GFX11TRUE16-NEXT: v_bfe_u32 v65, v27, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v35
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v28, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v35, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v28
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v8, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11TRUE16-NEXT: v_add3_u32 v148, v12, v28, 0x7fff
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v37, v50, v49
-; GFX11TRUE16-NEXT: v_or_b32_e32 v55, 0x400000, v36
-; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v27
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v10
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_bfe_u32 v69, v26, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v26
+; GFX11TRUE16-NEXT: v_bfe_u32 v81, v25, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v49, v66, v65
-; GFX11TRUE16-NEXT: v_bfe_u32 v65, v37, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v37
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v38, v52, v51
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v148, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v37, 0x7fff
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v51, v70, v69
-; GFX11TRUE16-NEXT: v_bfe_u32 v69, v38, 16, 1
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v52, v80, v71
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v149, v55, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v25
+; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v99, v6, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v101, v22, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v22
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v38
-; GFX11TRUE16-NEXT: v_bfe_u32 v71, v25, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v38, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v25
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v150, v64, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v25, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v83, v24, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v85, v48, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v65, v66, vcc_lo
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v50, v68, v67
-; GFX11TRUE16-NEXT: v_bfe_u32 v67, v26, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v26
+; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v113, v21, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v115, v4, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v117, v20, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v48
-; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v26, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v48, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v97, v49, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v49
-; GFX11TRUE16-NEXT: v_bfe_u32 v101, v50, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v67, v68, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v49, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v50
-; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v50, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v113, v51, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v69, v70, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v51
-; GFX11TRUE16-NEXT: v_bfe_u32 v117, v52, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v51, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v52
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v71, v80, vcc_lo
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v53, v82, v81
-; GFX11TRUE16-NEXT: v_bfe_u32 v81, v39, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v39
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v52, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v129, v53, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v39, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v53
-; GFX11TRUE16-NEXT: v_bfe_u32 v131, v18, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v53, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v81, v82, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v147, v0, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v83, v84, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v133, v54, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; GFX11TRUE16-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v85, v86, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.h, v25.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.h, v26.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.h, v27.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v54
-; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v54, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v87, v96, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v16.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v24.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v97, v98, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.h, v28.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.h, v29.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.h, v30.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v24.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.h, v16.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v25.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v26.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v27.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v29.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v99, v100, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v30.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v101, v102, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v23.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v103, v112, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v21.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v113, v114, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v22.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v115, v116, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v117, v118, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v21.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v119, v128, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v20.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v131, v132, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v18.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v135, v144, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v147, v33, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v133, v134, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v19.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v28.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v145, v146, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v33.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v18.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v32
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_mul_f32 v15, v15, v17 :: v_dual_and_b32 v18, 0xffff0000, v32
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v17, v31, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v15
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v15, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v32
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v17.l
+; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v32
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v15, v15, v18
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v17, v31, v17
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v15
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v17
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v15, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.h, v17.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmul_v32bf16:
@@ -18585,69 +18628,37 @@ define bfloat @v_fdiv_bf16(bfloat %a, bfloat %b) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fdiv_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_div_scale_f32 v2, null, v1, v1, v0
-; GFX11TRUE16-NEXT: v_rcp_f32_e32 v3, v2
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_fma_f32 v4, -v2, v3, 1.0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v4, v3
-; GFX11TRUE16-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v1, v0
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v4, v5, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fma_f32 v6, -v2, v4, v5
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v6, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fma_f32 v2, -v2, v4, v5
-; GFX11TRUE16-NEXT: v_div_fmas_f32 v2, v2, v3, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_div_fixup_f32 v0, v2, v1, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fdiv_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_div_scale_f32 v2, null, v1, v1, v0
-; GFX11FAKE16-NEXT: v_rcp_f32_e32 v3, v2
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_fma_f32 v4, -v2, v3, 1.0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fmac_f32_e32 v3, v4, v3
-; GFX11FAKE16-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v1, v0
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v4, v5, v3
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fma_f32 v6, -v2, v4, v5
-; GFX11FAKE16-NEXT: v_fmac_f32_e32 v4, v6, v3
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fma_f32 v2, -v2, v4, v5
-; GFX11FAKE16-NEXT: v_div_fmas_f32 v2, v2, v3, v4
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_div_fixup_f32 v0, v2, v1, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_fdiv_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_div_scale_f32 v2, null, v1, v1, v0
+; GFX11-NEXT: v_rcp_f32_e32 v3, v2
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_f32 v4, -v2, v3, 1.0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v3, v4, v3
+; GFX11-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v1, v0
+; GFX11-NEXT: v_mul_f32_e32 v4, v5, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v6, -v2, v4, v5
+; GFX11-NEXT: v_fmac_f32_e32 v4, v6, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v2, -v2, v4, v5
+; GFX11-NEXT: v_div_fmas_f32 v2, v2, v3, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_div_fixup_f32 v0, v2, v1, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_fdiv_bf16:
; GFX1250: ; %bb.0:
@@ -19106,39 +19117,22 @@ define bfloat @v_minnum_bf16(bfloat %a, bfloat %b) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_minnum_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_minnum_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_min_f32_e32 v0, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_minnum_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_minnum_bf16:
; GFX1250: ; %bb.0:
@@ -19271,10 +19265,10 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_minnum_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_min_f32_e32 v2, v3, v2
@@ -19286,12 +19280,14 @@ define <2 x bfloat> @v_minnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v2bf16:
@@ -19496,36 +19492,38 @@ define <3 x bfloat> @v_minnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_minnum_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_min_f32_e32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v3bf16:
@@ -19779,41 +19777,49 @@ define <4 x bfloat> @v_minnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_minnum_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_min_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_min_f32_e32 v1, v1, v3
; GFX11TRUE16-NEXT: v_dual_min_f32 v3, v7, v6 :: v_dual_min_f32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v4bf16:
@@ -20249,77 +20255,86 @@ define <8 x bfloat> @v_minnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v7, 16, v7
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_min_f32_e32 v7, v3, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v9, v10, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v2, v2, v6 :: v_dual_min_f32 v3, v3, v7
+; GFX11TRUE16-NEXT: v_min_f32_e32 v7, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v8
; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v8, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_min_f32_e32 v6, v2, v6
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v11, v7, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v11, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v6, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v7, v12, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v7, v9 :: v_dual_and_b32 v9, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.l
+; GFX11TRUE16-NEXT: v_dual_min_f32 v7, v11, v10 :: v_dual_and_b32 v10, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_min_f32_e32 v1, v1, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v5, v12, v11
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11TRUE16-NEXT: v_dual_min_f32 v0, v0, v4 :: v_dual_min_f32 v9, v14, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11TRUE16-NEXT: v_min_f32_e32 v5, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v11, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v10, v11 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v12, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v8bf16:
@@ -21136,152 +21151,175 @@ define <16 x bfloat> @v_minnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v19, v18 :: v_dual_and_b32 v21, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_min_f32_e32 v16, v17, v16
-; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v7, v7, v15
+; GFX11TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v14
+; GFX11TRUE16-NEXT: v_dual_min_f32 v15, v19, v15 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v6, v6, v14
+; GFX11TRUE16-NEXT: v_min_f32_e32 v16, v17, v16
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v16
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v14, v6, v14 :: v_dual_min_f32 v15, v7, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v17, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v16
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v16, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v17, v18, vcc_lo
; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v15, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v19, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v20, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v15
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v16.l
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v6
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v6, v17, v18 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_min_f32_e32 v14, v19, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v14
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v16, v20 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v15.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v14, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v17, v18 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v5, v5, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v12
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v14.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v12, v4, v12 :: v_dual_min_f32 v13, v5, v13
-; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v19, v18 :: v_dual_min_f32 v17, v22, v21
-; GFX11TRUE16-NEXT: v_bfe_u32 v22, v12, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v13, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v20, v18, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v21, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v16 :: v_dual_and_b32 v20, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v19, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v5, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v19, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_dual_min_f32 v13, v20, v13 :: v_dual_and_b32 v20, 0xffff0000, v11
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v13
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v22, v12, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_min_f32_e32 v3, v3, v11
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v14.l
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v13, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v13, v17, v22 :: v_dual_min_f32 v4, v4, v12
+; GFX11TRUE16-NEXT: v_dual_min_f32 v12, v21, v20 :: v_dual_and_b32 v21, 0xffff0000, v10
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v17, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_min_f32_e32 v12, v22, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v3, 16, 1
+; GFX11TRUE16-NEXT: v_min_f32_e32 v2, v2, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v17, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v3
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v2, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v17, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v12, v20, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v13.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v11.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v12, v17 :: v_dual_and_b32 v17, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v10.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v12, v19, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v16, v20, v19
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v21, v20 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v9
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v12, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_f32_e32 v1, v1, v9
-; GFX11TRUE16-NEXT: v_dual_min_f32 v10, v2, v10 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v11, v3, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v16
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v11, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v20, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v19, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v10, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v2, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v10, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v17, v19 :: v_dual_lshlrev_b32 v19, 16, v8
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v11.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v9, v20, v19
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v18, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
-; GFX11TRUE16-NEXT: v_dual_min_f32 v17, v22, v21 :: v_dual_min_f32 v0, v0, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX11TRUE16-NEXT: v_min_f32_e32 v9, v18, v17
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v9, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v18, v21, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v0, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v20, v21, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v19, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v9, v18, v19 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v8
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v20, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v17, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v16bf16:
@@ -22870,274 +22908,317 @@ define <32 x bfloat> @v_minnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32
-; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
-; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
-; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v20, v4, v20 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_min_f32_e32 v5, v5, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
; GFX11TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v17
; GFX11TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_bfe_u32 v103, v5, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v16
+; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
+; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_dual_min_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v3, v3, v19
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
+; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
+; GFX11TRUE16-NEXT: v_bfe_u32 v119, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_min_f32 v19, v82, v81 :: v_dual_lshlrev_b32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v129, v19, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v19
+; GFX11TRUE16-NEXT: v_min_f32_e32 v2, v2, v18
+; GFX11TRUE16-NEXT: v_dual_min_f32 v18, v84, v83 :: v_dual_min_f32 v9, v9, v25
+; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v131, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v2
+; GFX11TRUE16-NEXT: v_min_f32_e32 v17, v86, v85
+; GFX11TRUE16-NEXT: v_dual_min_f32 v8, v8, v24 :: v_dual_and_b32 v39, 0xffff0000, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v133, v18, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
+; GFX11TRUE16-NEXT: v_bfe_u32 v83, v8, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v8
+; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX11TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11TRUE16-NEXT: v_bfe_u32 v115, v20, 16, 1
; GFX11TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; GFX11TRUE16-NEXT: v_dual_min_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
-; GFX11TRUE16-NEXT: v_dual_min_f32 v23, v7, v23 :: v_dual_and_b32 v82, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_min_f32_e32 v22, v6, v22
-; GFX11TRUE16-NEXT: v_min_f32_e32 v17, v86, v85
-; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v20, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
-; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v27
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11TRUE16-NEXT: v_bfe_u32 v87, v23, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v99, v22, 16, 1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
-; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v23
-; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v22
-; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v23, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; GFX11TRUE16-NEXT: v_dual_min_f32 v21, v5, v21 :: v_dual_lshlrev_b32 v26, 16, v26
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v19, v3, v19 :: v_dual_and_b32 v52, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_min_f32 v27, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; GFX11TRUE16-NEXT: v_dual_min_f32 v26, v10, v26 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11TRUE16-NEXT: v_dual_min_f32 v24, v8, v24 :: v_dual_and_b32 v35, 0xffff0000, v29
-; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
+; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
+; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v16
+; GFX11TRUE16-NEXT: v_dual_min_f32 v24, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
+; GFX11TRUE16-NEXT: v_min_f32_e32 v7, v7, v23
+; GFX11TRUE16-NEXT: v_dual_min_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v85, v24, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v97, v23, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v23
+; GFX11TRUE16-NEXT: v_bfe_u32 v87, v7, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_min_f32_e32 v4, v4, v20
+; GFX11TRUE16-NEXT: v_min_f32_e32 v20, v80, v71
+; GFX11TRUE16-NEXT: v_bfe_u32 v71, v9, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v9
+; GFX11TRUE16-NEXT: v_dual_min_f32 v21, v70, v69 :: v_dual_lshlrev_b32 v10, 16, v10
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_dual_min_f32 v10, v10, v26 :: v_dual_lshlrev_b32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_dual_min_f32 v26, v52, v51 :: v_dual_min_f32 v25, v54, v53
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v6, v6, v22
+; GFX11TRUE16-NEXT: v_dual_min_f32 v11, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
; GFX11TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_bfe_u32 v103, v21, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v119, v19, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v29, v13, v29 :: v_dual_and_b32 v38, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_dual_min_f32 v33, v34, v33 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11TRUE16-NEXT: v_min_f32_e32 v34, v36, v35
-; GFX11TRUE16-NEXT: v_min_f32_e32 v36, v48, v39
-; GFX11TRUE16-NEXT: v_min_f32_e32 v0, v0, v16
-; GFX11TRUE16-NEXT: v_dual_min_f32 v48, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
+; GFX11TRUE16-NEXT: v_min_f32_e32 v22, v68, v67
+; GFX11TRUE16-NEXT: v_dual_min_f32 v27, v50, v49 :: v_dual_and_b32 v38, 0xffff0000, v12
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v21
-; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v19
-; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v21, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v19, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11TRUE16-NEXT: v_min_f32_e32 v35, v38, v37
-; GFX11TRUE16-NEXT: v_dual_min_f32 v30, v14, v30 :: v_dual_and_b32 v31, 0xffff0000, v15
+; GFX11TRUE16-NEXT: v_dual_min_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_min_f32 v29, v38, v37 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
+; GFX11TRUE16-NEXT: v_dual_min_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v14, v14, v30
+; GFX11TRUE16-NEXT: v_min_f32_e32 v28, v48, v39
+; GFX11TRUE16-NEXT: v_dual_min_f32 v30, v36, v35 :: v_dual_min_f32 v33, v34, v33
+; GFX11TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v35, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v14
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_f32_e32 v18, v2, v18
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v33, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v33
-; GFX11TRUE16-NEXT: v_bfe_u32 v14, v27, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v30, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v33
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v33, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v30
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v34, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v30, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v150, v14, v27, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v2, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v13
+; GFX11TRUE16-NEXT: v_bfe_u32 v49, v29, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v29
+; GFX11TRUE16-NEXT: v_bfe_u32 v51, v12, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11TRUE16-NEXT: v_dual_min_f32 v39, v54, v53 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_dual_min_f32 v54, v84, v83 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v34
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v29, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v36, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v34, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v4, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11TRUE16-NEXT: v_min_f32_e32 v25, v9, v25
-; GFX11TRUE16-NEXT: v_dual_min_f32 v28, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v29
-; GFX11TRUE16-NEXT: v_bfe_u32 v10, v35, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v29, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v149, v13, v36, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v53, v28, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v28
+; GFX11TRUE16-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11TRUE16-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v11
+; GFX11TRUE16-NEXT: v_bfe_u32 v65, v27, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v35
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v28, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v35, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v28
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v8, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11TRUE16-NEXT: v_add3_u32 v148, v12, v28, 0x7fff
-; GFX11TRUE16-NEXT: v_min_f32_e32 v37, v50, v49
-; GFX11TRUE16-NEXT: v_or_b32_e32 v55, 0x400000, v36
-; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v27
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v10
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_bfe_u32 v69, v26, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v26
+; GFX11TRUE16-NEXT: v_bfe_u32 v81, v25, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11TRUE16-NEXT: v_min_f32_e32 v49, v66, v65
-; GFX11TRUE16-NEXT: v_bfe_u32 v65, v37, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v37
-; GFX11TRUE16-NEXT: v_min_f32_e32 v38, v52, v51
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v148, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v37, 0x7fff
-; GFX11TRUE16-NEXT: v_min_f32_e32 v51, v70, v69
-; GFX11TRUE16-NEXT: v_bfe_u32 v69, v38, 16, 1
-; GFX11TRUE16-NEXT: v_min_f32_e32 v52, v80, v71
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v149, v55, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v25
+; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v99, v6, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v101, v22, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v22
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v38
-; GFX11TRUE16-NEXT: v_bfe_u32 v71, v25, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v38, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v25
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v150, v64, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v25, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v83, v24, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v85, v48, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v65, v66, vcc_lo
-; GFX11TRUE16-NEXT: v_min_f32_e32 v50, v68, v67
-; GFX11TRUE16-NEXT: v_bfe_u32 v67, v26, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v26
+; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v113, v21, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v115, v4, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v117, v20, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v48
-; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v26, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v48, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v97, v49, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v49
-; GFX11TRUE16-NEXT: v_bfe_u32 v101, v50, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v67, v68, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v49, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v50
-; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v50, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v113, v51, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v69, v70, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v51
-; GFX11TRUE16-NEXT: v_bfe_u32 v117, v52, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v51, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v52
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v71, v80, vcc_lo
-; GFX11TRUE16-NEXT: v_min_f32_e32 v53, v82, v81
-; GFX11TRUE16-NEXT: v_bfe_u32 v81, v39, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v39
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v52, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v129, v53, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v39, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v53
-; GFX11TRUE16-NEXT: v_bfe_u32 v131, v18, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v53, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v81, v82, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v147, v0, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v83, v84, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v133, v54, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; GFX11TRUE16-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v85, v86, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.h, v25.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.h, v26.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.h, v27.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v54
-; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v54, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v87, v96, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v16.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v24.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v97, v98, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.h, v28.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.h, v29.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.h, v30.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v24.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.h, v16.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v25.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v26.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v27.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v29.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v99, v100, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v30.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v101, v102, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v23.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v103, v112, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v21.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v113, v114, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v22.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v115, v116, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v117, v118, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v21.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v119, v128, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v20.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v131, v132, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v18.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v135, v144, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v147, v33, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v133, v134, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v19.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v28.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v145, v146, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v33.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v18.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v32
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_min_f32 v15, v15, v17 :: v_dual_and_b32 v18, 0xffff0000, v32
-; GFX11TRUE16-NEXT: v_min_f32_e32 v17, v31, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v15
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v15, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v32
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v17.l
+; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v32
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_min_f32_e32 v15, v15, v18
+; GFX11TRUE16-NEXT: v_min_f32_e32 v17, v31, v17
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v15
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v17
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v15, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.h, v17.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_minnum_v32bf16:
@@ -23531,39 +23612,22 @@ define bfloat @v_maxnum_bf16(bfloat %a, bfloat %b) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_maxnum_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_maxnum_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_max_f32_e32 v0, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_maxnum_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_maxnum_bf16:
; GFX1250: ; %bb.0:
@@ -23696,10 +23760,10 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_maxnum_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v1
; GFX11TRUE16-NEXT: v_max_f32_e32 v2, v3, v2
@@ -23711,12 +23775,14 @@ define <2 x bfloat> @v_maxnum_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v2bf16:
@@ -23921,36 +23987,38 @@ define <3 x bfloat> @v_maxnum_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_maxnum_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_max_f32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_max_f32_e32 v1, v1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v3bf16:
@@ -24204,41 +24272,49 @@ define <4 x bfloat> @v_maxnum_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b) #0 {
; GFX11TRUE16-LABEL: v_maxnum_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_dual_max_f32 v0, v0, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_max_f32_e32 v1, v1, v3
; GFX11TRUE16-NEXT: v_dual_max_f32 v3, v7, v6 :: v_dual_max_f32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v4bf16:
@@ -24674,77 +24750,86 @@ define <8 x bfloat> @v_maxnum_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b) #0 {
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v8, v9, v8 :: v_dual_lshlrev_b32 v7, 16, v7
; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_max_f32_e32 v7, v3, v7
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v1, v1, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v9, v10, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v2, v2, v6 :: v_dual_max_f32 v3, v3, v7
+; GFX11TRUE16-NEXT: v_max_f32_e32 v7, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v8
; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v8, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_max_f32_e32 v6, v2, v6
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v11, v7, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v11, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v6, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v12, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v2, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v7, v12, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v7, v9 :: v_dual_and_b32 v9, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.l
+; GFX11TRUE16-NEXT: v_dual_max_f32 v7, v11, v10 :: v_dual_and_b32 v10, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_max_f32_e32 v1, v1, v5
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v5, v12, v11
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11TRUE16-NEXT: v_dual_max_f32 v0, v0, v4 :: v_dual_max_f32 v9, v14, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11TRUE16-NEXT: v_max_f32_e32 v5, v10, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v5, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v11, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v13, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v10, v11 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v12, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v8bf16:
@@ -25561,152 +25646,175 @@ define <16 x bfloat> @v_maxnum_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v19, v18 :: v_dual_and_b32 v21, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_max_f32_e32 v16, v17, v16
-; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v7, v7, v15
+; GFX11TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v14
+; GFX11TRUE16-NEXT: v_dual_max_f32 v15, v19, v15 :: v_dual_lshlrev_b32 v14, 16, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v6, v6, v14
+; GFX11TRUE16-NEXT: v_max_f32_e32 v16, v17, v16
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v17, v16, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v16
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v14, v6, v14 :: v_dual_max_f32 v15, v7, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v17, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v16
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v16, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v7
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v17, v18, vcc_lo
; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v15, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v19, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v16, v20, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v14, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v15
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v16.l
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v17, v18, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v6
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v6, v17, v18 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_max_f32_e32 v14, v19, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v14
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v14, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v16, v20 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v15.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v14, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v14, v17, v18 :: v_dual_lshlrev_b32 v13, 16, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v5, v5, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v12
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v14.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v12, v4, v12 :: v_dual_max_f32 v13, v5, v13
-; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v19, v18 :: v_dual_max_f32 v17, v22, v21
-; GFX11TRUE16-NEXT: v_bfe_u32 v22, v12, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v13, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v20, v18, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v21, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
-; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v18
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v5, v5, v16 :: v_dual_and_b32 v20, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v19, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v5, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v19, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_dual_max_f32 v13, v20, v13 :: v_dual_and_b32 v20, 0xffff0000, v11
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v13
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v22, v12, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_max_f32_e32 v3, v3, v11
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v14.l
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v13, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v13, v17, v22 :: v_dual_max_f32 v4, v4, v12
+; GFX11TRUE16-NEXT: v_dual_max_f32 v12, v21, v20 :: v_dual_and_b32 v21, 0xffff0000, v10
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v4, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v17, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_max_f32_e32 v12, v22, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v3, 16, 1
+; GFX11TRUE16-NEXT: v_max_f32_e32 v2, v2, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v12, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v17, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v3
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v2, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v17, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v12, v20, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v13.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v11.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v12, v17 :: v_dual_and_b32 v17, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v10.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v12, v19, v18
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v16, v20, v19
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v12.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v16, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v16, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v21, v20 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v9
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v18, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v12, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_max_f32_e32 v1, v1, v9
-; GFX11TRUE16-NEXT: v_dual_max_f32 v10, v2, v10 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v11, v3, v11
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v16
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v11, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v11
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_add3_u32 v17, v20, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v19, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v18
-; GFX11TRUE16-NEXT: v_bfe_u32 v20, v10, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v2, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v20, v10, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v17, v19 :: v_dual_lshlrev_b32 v19, 16, v8
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v11.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v9, v20, v19
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v16, v18, vcc_lo
-; GFX11TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
-; GFX11TRUE16-NEXT: v_dual_max_f32 v17, v22, v21 :: v_dual_max_f32 v0, v0, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
+; GFX11TRUE16-NEXT: v_max_f32_e32 v9, v18, v17
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v19, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v17, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v9, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v17, v18, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_add3_u32 v18, v21, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v0, 16, 1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v20, v21, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v19, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v9, v18, v19 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v8
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v17, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v17, v17, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v20, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v20, v21, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v17, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v16bf16:
@@ -27295,274 +27403,317 @@ define <32 x bfloat> @v_maxnum_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32
-; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
-; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
-; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v20, v4, v20 :: v_dual_lshlrev_b32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_max_f32_e32 v5, v5, v21
+; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
; GFX11TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v17
; GFX11TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_bfe_u32 v103, v5, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v16
+; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
+; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_dual_max_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v3, v3, v19
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v22
+; GFX11TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v11
+; GFX11TRUE16-NEXT: v_bfe_u32 v119, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_dual_max_f32 v19, v82, v81 :: v_dual_lshlrev_b32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v129, v19, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v19
+; GFX11TRUE16-NEXT: v_max_f32_e32 v2, v2, v18
+; GFX11TRUE16-NEXT: v_dual_max_f32 v18, v84, v83 :: v_dual_max_f32 v9, v9, v25
+; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v19, 0x7fff
; GFX11TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v131, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v2
+; GFX11TRUE16-NEXT: v_max_f32_e32 v17, v86, v85
+; GFX11TRUE16-NEXT: v_dual_max_f32 v8, v8, v24 :: v_dual_and_b32 v39, 0xffff0000, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v133, v18, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v2, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
+; GFX11TRUE16-NEXT: v_bfe_u32 v83, v8, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v8
+; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v18
+; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX11TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v23
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11TRUE16-NEXT: v_bfe_u32 v115, v20, 16, 1
; GFX11TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v24
-; GFX11TRUE16-NEXT: v_dual_max_f32 v1, v1, v17 :: v_dual_lshlrev_b32 v24, 16, v24
-; GFX11TRUE16-NEXT: v_dual_max_f32 v23, v7, v23 :: v_dual_and_b32 v82, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_max_f32_e32 v22, v6, v22
-; GFX11TRUE16-NEXT: v_max_f32_e32 v17, v86, v85
-; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v20, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v18
-; GFX11TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v27
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11TRUE16-NEXT: v_bfe_u32 v87, v23, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v99, v22, 16, 1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v5
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v19
-; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v23
-; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v22
-; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v23, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v26
-; GFX11TRUE16-NEXT: v_dual_max_f32 v21, v5, v21 :: v_dual_lshlrev_b32 v26, 16, v26
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v19, v3, v19 :: v_dual_and_b32 v52, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_dual_max_f32 v27, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v25
-; GFX11TRUE16-NEXT: v_dual_max_f32 v26, v10, v26 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11TRUE16-NEXT: v_dual_max_f32 v24, v8, v24 :: v_dual_and_b32 v35, 0xffff0000, v29
-; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
+; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v8, 0x7fff
+; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v16
+; GFX11TRUE16-NEXT: v_dual_max_f32 v24, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
+; GFX11TRUE16-NEXT: v_max_f32_e32 v7, v7, v23
+; GFX11TRUE16-NEXT: v_dual_max_f32 v23, v66, v65 :: v_dual_lshlrev_b32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v29
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v85, v24, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v97, v23, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v23
+; GFX11TRUE16-NEXT: v_bfe_u32 v87, v7, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v20
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v23, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v7
+; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_max_f32_e32 v4, v4, v20
+; GFX11TRUE16-NEXT: v_max_f32_e32 v20, v80, v71
+; GFX11TRUE16-NEXT: v_bfe_u32 v71, v9, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v9
+; GFX11TRUE16-NEXT: v_dual_max_f32 v21, v70, v69 :: v_dual_lshlrev_b32 v10, 16, v10
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_dual_max_f32 v10, v10, v26 :: v_dual_lshlrev_b32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_dual_max_f32 v26, v52, v51 :: v_dual_max_f32 v25, v54, v53
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v6, v6, v22
+; GFX11TRUE16-NEXT: v_dual_max_f32 v11, v11, v27 :: v_dual_and_b32 v36, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_and_b32_e32 v33, 0xffff0000, v30
; GFX11TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11TRUE16-NEXT: v_bfe_u32 v103, v21, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v119, v19, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v29, v13, v29 :: v_dual_and_b32 v38, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_dual_max_f32 v33, v34, v33 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11TRUE16-NEXT: v_max_f32_e32 v34, v36, v35
-; GFX11TRUE16-NEXT: v_max_f32_e32 v36, v48, v39
-; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v16
-; GFX11TRUE16-NEXT: v_dual_max_f32 v48, v64, v55 :: v_dual_and_b32 v37, 0xffff0000, v28
+; GFX11TRUE16-NEXT: v_max_f32_e32 v22, v68, v67
+; GFX11TRUE16-NEXT: v_dual_max_f32 v27, v50, v49 :: v_dual_and_b32 v38, 0xffff0000, v12
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v21
-; GFX11TRUE16-NEXT: v_or_b32_e32 v128, 0x400000, v19
-; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v21, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v19, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v25
-; GFX11TRUE16-NEXT: v_max_f32_e32 v35, v38, v37
-; GFX11TRUE16-NEXT: v_dual_max_f32 v30, v14, v30 :: v_dual_and_b32 v31, 0xffff0000, v15
+; GFX11TRUE16-NEXT: v_dual_max_f32 v13, v13, v29 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_max_f32 v29, v38, v37 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_and_b32_e32 v31, 0xffff0000, v15
+; GFX11TRUE16-NEXT: v_dual_max_f32 v12, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v14, v14, v30
+; GFX11TRUE16-NEXT: v_max_f32_e32 v28, v48, v39
+; GFX11TRUE16-NEXT: v_dual_max_f32 v30, v36, v35 :: v_dual_max_f32 v33, v34, v33
+; GFX11TRUE16-NEXT: v_bfe_u32 v39, v13, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v35, v14, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v36, 0x400000, v14
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v18, v2, v18
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v33, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v33
-; GFX11TRUE16-NEXT: v_bfe_u32 v14, v27, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v30, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v37, v30, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v16, v33, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v34, 0x400000, v33
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v33, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v30
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v34, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v30, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v150, v14, v27, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v2, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v35, v35, v14, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v38, 0x400000, v30
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v33, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v37, v37, v30, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v48, 0x400000, v13
+; GFX11TRUE16-NEXT: v_bfe_u32 v49, v29, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v39, v39, v13, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v34, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
+; GFX11TRUE16-NEXT: v_or_b32_e32 v50, 0x400000, v29
+; GFX11TRUE16-NEXT: v_bfe_u32 v51, v12, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v49, v49, v29, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v52, 0x400000, v12
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v35, v36, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v30, v30
-; GFX11TRUE16-NEXT: v_dual_max_f32 v39, v54, v53 :: v_dual_lshlrev_b32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_dual_max_f32 v54, v84, v83 :: v_dual_lshlrev_b32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v28
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v34
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v29, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v36, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v34, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v4, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11TRUE16-NEXT: v_max_f32_e32 v25, v9, v25
-; GFX11TRUE16-NEXT: v_dual_max_f32 v28, v12, v28 :: v_dual_lshlrev_b32 v15, 16, v15
-; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v29
-; GFX11TRUE16-NEXT: v_bfe_u32 v10, v35, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v29, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v149, v13, v36, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v53, v28, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v51, v51, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v54, 0x400000, v28
+; GFX11TRUE16-NEXT: v_bfe_u32 v55, v11, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v37, v38, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11TRUE16-NEXT: v_add3_u32 v53, v53, v28, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v11
+; GFX11TRUE16-NEXT: v_bfe_u32 v65, v27, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v55, v55, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v39, v48, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v35
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v28, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v35, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v28
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v8, v9, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11TRUE16-NEXT: v_add3_u32 v148, v12, v28, 0x7fff
-; GFX11TRUE16-NEXT: v_max_f32_e32 v37, v50, v49
-; GFX11TRUE16-NEXT: v_or_b32_e32 v55, 0x400000, v36
-; GFX11TRUE16-NEXT: v_or_b32_e32 v64, 0x400000, v27
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v27
+; GFX11TRUE16-NEXT: v_bfe_u32 v67, v10, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v27, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v10
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v49, v50, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11TRUE16-NEXT: v_bfe_u32 v69, v26, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v26
+; GFX11TRUE16-NEXT: v_bfe_u32 v81, v25, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v51, v52, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v28, v28
-; GFX11TRUE16-NEXT: v_max_f32_e32 v49, v66, v65
-; GFX11TRUE16-NEXT: v_bfe_u32 v65, v37, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v37
-; GFX11TRUE16-NEXT: v_max_f32_e32 v38, v52, v51
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v148, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v37, 0x7fff
-; GFX11TRUE16-NEXT: v_max_f32_e32 v51, v70, v69
-; GFX11TRUE16-NEXT: v_bfe_u32 v69, v38, 16, 1
-; GFX11TRUE16-NEXT: v_max_f32_e32 v52, v80, v71
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v149, v55, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v26, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v25
+; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v25, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v99, v6, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v53, v54, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v101, v22, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v22
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v55, v64, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
-; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v38
-; GFX11TRUE16-NEXT: v_bfe_u32 v71, v25, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v38, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v25
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v150, v64, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v25, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v83, v24, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v85, v48, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v65, v66, vcc_lo
-; GFX11TRUE16-NEXT: v_max_f32_e32 v50, v68, v67
-; GFX11TRUE16-NEXT: v_bfe_u32 v67, v26, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v26
+; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v22, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v113, v21, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v115, v4, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v65, v66, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v117, v20, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v67, v68, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v48
-; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v26, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v48, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v97, v49, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v49
-; GFX11TRUE16-NEXT: v_bfe_u32 v101, v50, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v67, v68, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v49, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v50
-; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v50, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v113, v51, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v69, v70, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
-; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v51
-; GFX11TRUE16-NEXT: v_bfe_u32 v117, v52, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v51, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v52
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v71, v80, vcc_lo
-; GFX11TRUE16-NEXT: v_max_f32_e32 v53, v82, v81
-; GFX11TRUE16-NEXT: v_bfe_u32 v81, v39, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v39
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v52, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v129, v53, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v39, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v130, 0x400000, v53
-; GFX11TRUE16-NEXT: v_bfe_u32 v131, v18, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v132, 0x400000, v18
-; GFX11TRUE16-NEXT: v_add3_u32 v129, v129, v53, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v81, v82, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_bfe_u32 v135, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v131, v131, v18, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v144, 0x400000, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v20
+; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v18, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v147, v0, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v83, v84, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX11TRUE16-NEXT: v_add3_u32 v135, v135, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v133, v54, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v69, v70, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11TRUE16-NEXT: v_or_b32_e32 v33, 0x400000, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v16
; GFX11TRUE16-NEXT: v_add3_u32 v147, v147, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v85, v86, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v71, v80, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v29
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v27
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v26
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v81, v82, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v25
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v83, v84, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v85, v86, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.h, v25.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.h, v26.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.h, v27.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v24
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v87, v96, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11TRUE16-NEXT: v_or_b32_e32 v134, 0x400000, v54
-; GFX11TRUE16-NEXT: v_bfe_u32 v145, v17, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v133, v133, v54, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v146, 0x400000, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v87, v96, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11TRUE16-NEXT: v_add3_u32 v145, v145, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v16.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v24.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v23.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v97, v98, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.h, v28.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.h, v29.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.h, v30.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v97, v98, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v24.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.h, v16.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v23
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v99, v100, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v25.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v26.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v27.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v29.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v99, v100, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v30.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v101, v102, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v101, v102, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v23.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v103, v112, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v103, v112, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v21.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v113, v114, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v113, v114, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v22.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v115, v116, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v115, v116, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v117, v118, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v117, v118, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v21.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v119, v128, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v119, v128, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v19.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v129, v130, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v20.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v19
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v131, v132, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v131, v132, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v133, v134, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v18.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v135, v144, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v147, v33, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v133, v134, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v19.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v18
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v135, v144, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v28.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v145, v146, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v33.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v145, v146, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v18.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v147, v33, vcc_lo
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v32
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_dual_max_f32 v15, v15, v17 :: v_dual_and_b32 v18, 0xffff0000, v32
-; GFX11TRUE16-NEXT: v_max_f32_e32 v17, v31, v18
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v15, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v17, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v15
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v15, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v32
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v17.l
+; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v32
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_max_f32_e32 v15, v15, v18
+; GFX11TRUE16-NEXT: v_max_f32_e32 v17, v31, v17
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v15
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v17, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v17
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v15, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v18, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v19, v21, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v15
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.h, v17.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_maxnum_v32bf16:
@@ -28044,79 +28195,42 @@ define bfloat @v_sqrt_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_sqrt_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_sqrt_f32_e32 v1, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v1
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v3, 1, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v4, -v2, v1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_fma_f32 v5, -v3, v1, v0
-; GFX11TRUE16-NEXT: v_cmp_ge_f32_e64 s0, 0, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
-; GFX11TRUE16-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_sqrt_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_sqrt_f32_e32 v1, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v2, -1, v1
-; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v3, 1, v1
-; GFX11FAKE16-NEXT: v_fma_f32 v4, -v2, v1, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_fma_f32 v5, -v3, v1, v0
-; GFX11FAKE16-NEXT: v_cmp_ge_f32_e64 s0, 0, v4
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
-; GFX11FAKE16-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_sqrt_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
+; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_sqrt_f32_e32 v1, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v2, -1, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 1, v1
+; GFX11-NEXT: v_fma_f32 v4, -v2, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_fma_f32 v5, -v3, v1, v0
+; GFX11-NEXT: v_cmp_ge_f32_e64 s0, 0, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
+; GFX11-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
+; GFX11-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
+; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_sqrt_bf16:
; GFX1250TRUE16: ; %bb.0:
@@ -28397,129 +28511,67 @@ define bfloat @v_rsq_bf16(bfloat %x) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_rsq_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_sqrt_f32_e32 v1, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v1
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v3, 1, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v4, -v2, v1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_fma_f32 v5, -v3, v1, v0
-; GFX11TRUE16-NEXT: v_cmp_ge_f32_e64 s0, 0, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
-; GFX11TRUE16-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_div_scale_f32 v1, null, v0, v0, 1.0
-; GFX11TRUE16-NEXT: v_div_scale_f32 v4, vcc_lo, 1.0, v0, 1.0
-; GFX11TRUE16-NEXT: v_rcp_f32_e32 v2, v1
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_fma_f32 v3, -v1, v2, 1.0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v3, v4, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fma_f32 v5, -v1, v3, v4
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v5, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fma_f32 v1, -v1, v3, v4
-; GFX11TRUE16-NEXT: v_div_fmas_f32 v1, v1, v2, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_div_fixup_f32 v0, v1, v0, 1.0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_rsq_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_sqrt_f32_e32 v1, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v2, -1, v1
-; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v3, 1, v1
-; GFX11FAKE16-NEXT: v_fma_f32 v4, -v2, v1, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_fma_f32 v5, -v3, v1, v0
-; GFX11FAKE16-NEXT: v_cmp_ge_f32_e64 s0, 0, v4
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
-; GFX11FAKE16-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_div_scale_f32 v1, null, v0, v0, 1.0
-; GFX11FAKE16-NEXT: v_div_scale_f32 v4, vcc_lo, 1.0, v0, 1.0
-; GFX11FAKE16-NEXT: v_rcp_f32_e32 v2, v1
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_fma_f32 v3, -v1, v2, 1.0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fmac_f32_e32 v2, v3, v2
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v3, v4, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fma_f32 v5, -v1, v3, v4
-; GFX11FAKE16-NEXT: v_fmac_f32_e32 v3, v5, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fma_f32 v1, -v1, v3, v4
-; GFX11FAKE16-NEXT: v_div_fmas_f32 v1, v1, v2, v3
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_div_fixup_f32 v0, v1, v0, 1.0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_rsq_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
+; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_sqrt_f32_e32 v1, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v2, -1, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 1, v1
+; GFX11-NEXT: v_fma_f32 v4, -v2, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_fma_f32 v5, -v3, v1, v0
+; GFX11-NEXT: v_cmp_ge_f32_e64 s0, 0, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
+; GFX11-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
+; GFX11-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
+; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_div_scale_f32 v1, null, v0, v0, 1.0
+; GFX11-NEXT: v_div_scale_f32 v4, vcc_lo, 1.0, v0, 1.0
+; GFX11-NEXT: v_rcp_f32_e32 v2, v1
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_f32 v3, -v1, v2, 1.0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v2, v3, v2
+; GFX11-NEXT: v_mul_f32_e32 v3, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v5, -v1, v3, v4
+; GFX11-NEXT: v_fmac_f32_e32 v3, v5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v1, -v1, v3, v4
+; GFX11-NEXT: v_div_fmas_f32 v1, v1, v2, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_div_fixup_f32 v0, v1, v0, 1.0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_rsq_bf16:
; GFX1250TRUE16: ; %bb.0:
@@ -28801,129 +28853,67 @@ define bfloat @v_neg_rsq_bf16(bfloat %x) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_neg_rsq_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_sqrt_f32_e32 v1, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v1
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v3, 1, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v4, -v2, v1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_fma_f32 v5, -v3, v1, v0
-; GFX11TRUE16-NEXT: v_cmp_ge_f32_e64 s0, 0, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
-; GFX11TRUE16-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_div_scale_f32 v1, null, v0, v0, -1.0
-; GFX11TRUE16-NEXT: v_div_scale_f32 v4, vcc_lo, -1.0, v0, -1.0
-; GFX11TRUE16-NEXT: v_rcp_f32_e32 v2, v1
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_fma_f32 v3, -v1, v2, 1.0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v3, v4, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fma_f32 v5, -v1, v3, v4
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v5, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fma_f32 v1, -v1, v3, v4
-; GFX11TRUE16-NEXT: v_div_fmas_f32 v1, v1, v2, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_div_fixup_f32 v0, v1, v0, -1.0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_neg_rsq_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_sqrt_f32_e32 v1, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v2, -1, v1
-; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v3, 1, v1
-; GFX11FAKE16-NEXT: v_fma_f32 v4, -v2, v1, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_fma_f32 v5, -v3, v1, v0
-; GFX11FAKE16-NEXT: v_cmp_ge_f32_e64 s0, 0, v4
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
-; GFX11FAKE16-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_div_scale_f32 v1, null, v0, v0, -1.0
-; GFX11FAKE16-NEXT: v_div_scale_f32 v4, vcc_lo, -1.0, v0, -1.0
-; GFX11FAKE16-NEXT: v_rcp_f32_e32 v2, v1
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_fma_f32 v3, -v1, v2, 1.0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fmac_f32_e32 v2, v3, v2
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v3, v4, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fma_f32 v5, -v1, v3, v4
-; GFX11FAKE16-NEXT: v_fmac_f32_e32 v3, v5, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fma_f32 v1, -v1, v3, v4
-; GFX11FAKE16-NEXT: v_div_fmas_f32 v1, v1, v2, v3
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_div_fixup_f32 v0, v1, v0, -1.0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_neg_rsq_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x4f800000, v0
+; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xf800000, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_sqrt_f32_e32 v1, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_add_nc_u32_e32 v2, -1, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v3, 1, v1
+; GFX11-NEXT: v_fma_f32 v4, -v2, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_fma_f32 v5, -v3, v1, v0
+; GFX11-NEXT: v_cmp_ge_f32_e64 s0, 0, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
+; GFX11-NEXT: v_cmp_lt_f32_e64 s0, 0, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v3, s0
+; GFX11-NEXT: v_mul_f32_e32 v2, 0x37800000, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
+; GFX11-NEXT: v_cmp_class_f32_e64 vcc_lo, v0, 0x260
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_div_scale_f32 v1, null, v0, v0, -1.0
+; GFX11-NEXT: v_div_scale_f32 v4, vcc_lo, -1.0, v0, -1.0
+; GFX11-NEXT: v_rcp_f32_e32 v2, v1
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_fma_f32 v3, -v1, v2, 1.0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v2, v3, v2
+; GFX11-NEXT: v_mul_f32_e32 v3, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v5, -v1, v3, v4
+; GFX11-NEXT: v_fmac_f32_e32 v3, v5, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f32 v1, -v1, v3, v4
+; GFX11-NEXT: v_div_fmas_f32 v1, v1, v2, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_div_fixup_f32 v0, v1, v0, -1.0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_neg_rsq_bf16:
; GFX1250TRUE16: ; %bb.0:
@@ -29017,37 +29007,21 @@ define bfloat @v_ldexp_bf16_i32(bfloat %a, i32 %b) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_ldexp_bf16_i32:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_ldexp_bf16_i32:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_ldexp_bf16_i32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_ldexp_bf16_i32:
; GFX1250: ; %bb.0:
@@ -29141,50 +29115,46 @@ define { bfloat, i16 } @v_frexp_bf16_i16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_frexp_bf16_i16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_frexp_mant_f32_e32 v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_frexp_bf16_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_frexp_mant_f32_e32 v0, v1
+; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11FAKE16-LABEL: v_frexp_bf16_i16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_frexp_mant_f32_e32 v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1250TRUE16-LABEL: v_frexp_bf16_i16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_frexp_mant_f32_e32 v1, v0
+; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v1, s0
+; GFX1250TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX1250-LABEL: v_frexp_bf16_i16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_frexp_mant_f32_e32 v0, v1
-; GFX1250-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250FAKE16-LABEL: v_frexp_bf16_i16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250FAKE16-NEXT: v_frexp_mant_f32_e32 v0, v1
+; GFX1250FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = call { bfloat, i16 } @llvm.frexp.bf16.i16(bfloat %a)
ret { bfloat, i16 } %op
}
@@ -29356,67 +29326,36 @@ define bfloat @v_log_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_log_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
-; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x41b17218, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_log_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
-; GFX11FAKE16-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
-; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x41b17218, vcc_lo
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11FAKE16-NEXT: v_sub_f32_e32 v0, v0, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_log_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_log_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x3f317217, v0
+; GFX11-NEXT: v_fma_f32 v2, 0x3f317217, v0, -v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmamk_f32 v2, v0, 0x3377d1cf, v2
+; GFX11-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 0x41b17218, vcc_lo
+; GFX11-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_log_bf16:
; GFX1250: ; %bb.0:
@@ -29554,51 +29493,28 @@ define bfloat @v_log2_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_log2_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_log2_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_log2_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
+; GFX11-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_log_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_log2_bf16:
; GFX1250TRUE16: ; %bb.0:
@@ -29779,67 +29695,36 @@ define bfloat @v_log10_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_log10_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
-; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x411a209b, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_log10_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
-; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_log_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
-; GFX11FAKE16-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
-; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v2
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x411a209b, vcc_lo
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11FAKE16-NEXT: v_sub_f32_e32 v0, v0, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_log10_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_log_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x3e9a209a, v0
+; GFX11-NEXT: v_fma_f32 v2, 0x3e9a209a, v0, -v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmamk_f32 v2, v0, 0x3284fbcf, v2
+; GFX11-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 0x411a209b, vcc_lo
+; GFX11-NEXT: v_cmp_gt_f32_e64 vcc_lo, 0x7f800000, |v0|
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: v_sub_f32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_log10_bf16:
; GFX1250: ; %bb.0:
@@ -30039,99 +29924,64 @@ define bfloat @v_exp_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_exp_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v2, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v3, 0x3fb8aa3b, v0, -v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX11TRUE16-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3
-; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_exp_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; GFX11FAKE16-NEXT: v_rndne_f32_e32 v2, v1
-; GFX11FAKE16-NEXT: v_fma_f32 v3, 0x3fb8aa3b, v0, -v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX11FAKE16-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3
-; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX11FAKE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11FAKE16-NEXT: v_exp_f32_e32 v1, v1
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
-; GFX11FAKE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_exp_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; GFX11-NEXT: v_rndne_f32_e32 v2, v1
+; GFX11-NEXT: v_fma_f32 v3, 0x3fb8aa3b, v0, -v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX11-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3
+; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11-NEXT: v_exp_f32_e32 v1, v1
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_exp_bf16:
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x3fb8aa3b
-; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v1, s0, neg(0) op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v1, s0, -v2 op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v4, v2
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v0, s0, neg(0) op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v0, s0, -v1 op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v3, v1
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x32a5705f
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v1, s0, v3 op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v0, s0, v2 op_sel_hi:[1,0,0]
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v4
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1250TRUE16-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
; GFX1250TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1250TRUE16-NEXT: v_nop
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
; GFX1250TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -30279,51 +30129,28 @@ define bfloat @v_exp2_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_exp2_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_exp2_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX11FAKE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_exp_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_exp2_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_exp_f32_e32 v0, v0
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_exp2_bf16:
; GFX1250TRUE16: ; %bb.0:
@@ -30507,99 +30334,64 @@ define bfloat @v_exp10_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_exp10_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v2, v1
-; GFX11TRUE16-NEXT: v_fma_f32 v3, 0x40549a78, v0, -v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX11TRUE16-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3
-; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_exp10_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; GFX11FAKE16-NEXT: v_rndne_f32_e32 v2, v1
-; GFX11FAKE16-NEXT: v_fma_f32 v3, 0x40549a78, v0, -v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX11FAKE16-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3
-; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX11FAKE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11FAKE16-NEXT: v_exp_f32_e32 v1, v1
-; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11FAKE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
-; GFX11FAKE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_exp10_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; GFX11-NEXT: v_rndne_f32_e32 v2, v1
+; GFX11-NEXT: v_fma_f32 v3, 0x40549a78, v0, -v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX11-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3
+; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11-NEXT: v_exp_f32_e32 v1, v1
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_exp10_bf16:
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x40549a78
-; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v1, s0, neg(0) op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v3, v1, s0, -v2 op_sel_hi:[1,0,0]
-; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v4, v2
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v0, s0, neg(0) op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v0, s0, -v1 op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_rndne_f32_e32 v3, v1
; GFX1250TRUE16-NEXT: s_mov_b32 s0, 0x33979a37
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instid1(SALU_CYCLE_1)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v1, v1, s0, v3 op_sel_hi:[1,0,0]
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v2, v0, s0, v2 op_sel_hi:[1,0,0]
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_sub_f32_e32 v2, v2, v4
-; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v2, v1
-; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v4
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
+; GFX1250TRUE16-NEXT: v_dual_sub_f32 v1, v1, v3 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX1250TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX1250TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
; GFX1250TRUE16-NEXT: v_exp_f32_e32 v1, v1
; GFX1250TRUE16-NEXT: v_nop
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
; GFX1250TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -30704,37 +30496,21 @@ define bfloat @v_ceil_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_ceil_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_ceil_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_ceil_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_ceil_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_ceil_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_ceil_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_ceil_bf16:
; GFX1250: ; %bb.0:
@@ -30817,37 +30593,21 @@ define bfloat @v_trunc_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_trunc_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_trunc_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_trunc_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_trunc_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_trunc_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_trunc_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_trunc_bf16:
; GFX1250: ; %bb.0:
@@ -30930,37 +30690,21 @@ define bfloat @v_rint_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_rint_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_rint_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_rndne_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_rint_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_rint_bf16:
; GFX1250: ; %bb.0:
@@ -31043,37 +30787,21 @@ define bfloat @v_nearbyint_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_nearbyint_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_nearbyint_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_rndne_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_nearbyint_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_nearbyint_bf16:
; GFX1250: ; %bb.0:
@@ -31192,69 +30920,65 @@ define bfloat @v_round_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_round_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_trunc_f32_e32 v1, v0
-; GFX11TRUE16-NEXT: v_sub_f32_e32 v2, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cmp_ge_f32_e64 s0, |v2|, 0.5
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
-; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_round_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_trunc_f32_e32 v1, v0
+; GFX11-NEXT: v_sub_f32_e32 v2, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_ge_f32_e64 s0, |v2|, 0.5
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
+; GFX11-NEXT: v_add_f32_e32 v0, v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11FAKE16-LABEL: v_round_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_trunc_f32_e32 v1, v0
-; GFX11FAKE16-NEXT: v_sub_f32_e32 v2, v0, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cmp_ge_f32_e64 s0, |v2|, 0.5
-; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0
-; GFX11FAKE16-NEXT: v_add_f32_e32 v0, v1, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1250TRUE16-LABEL: v_round_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX1250TRUE16-NEXT: v_trunc_f32_e32 v2, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
+; GFX1250TRUE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX1250TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX1250-LABEL: v_round_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_trunc_f32_e32 v2, v1
-; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
-; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
-; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250FAKE16-LABEL: v_round_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_trunc_f32_e32 v2, v1
+; GFX1250FAKE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
+; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX1250FAKE16-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.round.bf16(bfloat %a)
ret bfloat %op
}
@@ -31327,37 +31051,21 @@ define bfloat @v_roundeven_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_roundeven_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_rndne_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_roundeven_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_rndne_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_roundeven_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_rndne_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_roundeven_bf16:
; GFX1250: ; %bb.0:
@@ -31440,37 +31148,21 @@ define bfloat @v_floor_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_floor_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_floor_f32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_floor_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_floor_f32_e32 v0, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_floor_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_floor_f32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_floor_bf16:
; GFX1250: ; %bb.0:
@@ -31553,37 +31245,21 @@ define bfloat @v_canonicalize_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_canonicalize_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_max_f32_e32 v0, v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_canonicalize_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_max_f32_e32 v0, v0, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_canonicalize_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v0, v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_canonicalize_bf16:
; GFX1250: ; %bb.0:
@@ -34734,37 +34410,21 @@ define bfloat @v_sitofp_i16_to_bf16(i16 %x) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_sitofp_i16_to_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_sitofp_i16_to_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_sitofp_i16_to_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_sitofp_i16_to_bf16:
; GFX1250: ; %bb.0:
@@ -34872,8 +34532,8 @@ define <2 x bfloat> @v_sitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX11TRUE16-LABEL: v_sitofp_v2i16_to_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 16
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v1, 16, v0
+; GFX11TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
@@ -34889,8 +34549,11 @@ define <2 x bfloat> @v_sitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v2i16_to_v2bf16:
@@ -35056,34 +34719,37 @@ define <3 x bfloat> @v_sitofp_v3i16_to_v3bf16(<3 x i16> %x) #0 {
; GFX11TRUE16-LABEL: v_sitofp_v3i16_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v2, 16, v0
+; GFX11TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
; GFX11TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i16_to_v3bf16:
@@ -35317,30 +34983,35 @@ define <4 x bfloat> @v_sitofp_v4i16_to_v4bf16(<4 x i16> %x) #0 {
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v3, v3
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v11, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v4i16_to_v4bf16:
@@ -35464,33 +35135,19 @@ define bfloat @v_sitofp_i32_to_bf16(i32 %x) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_sitofp_i32_to_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_sitofp_i32_to_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_sitofp_i32_to_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_sitofp_i32_to_bf16:
; GFX1250: ; %bb.0:
@@ -35593,22 +35250,25 @@ define <2 x bfloat> @v_sitofp_v2i32_to_v2bf16(<2 x i32> %x) #0 {
; GFX11TRUE16-LABEL: v_sitofp_v2i32_to_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v2i32_to_v2bf16:
@@ -35763,29 +35423,32 @@ define <3 x bfloat> @v_sitofp_v3i32_to_v3bf16(<3 x i32> %x) #0 {
; GFX11TRUE16-LABEL: v_sitofp_v3i32_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i32_to_v3bf16:
@@ -35987,33 +35650,39 @@ define <4 x bfloat> @v_sitofp_v4i32_to_v4bf16(<4 x i32> %x) #0 {
; GFX11TRUE16-LABEL: v_sitofp_v4i32_to_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v3, v3
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v3, v3
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v6, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v4i32_to_v4bf16:
@@ -36192,87 +35861,79 @@ define bfloat @v_sitofp_i64_to_bf16(i64 %x) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_sitofp_i64_to_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v2, v0, v1
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v3, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v2, 31, v2
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v3, -1, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v2, 32, v2
-; GFX11TRUE16-NEXT: v_min_u32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v2
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_sitofp_i64_to_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_xor_b32_e32 v2, v0, v1
+; GFX11-NEXT: v_cls_i32_e32 v3, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_ashrrev_i32_e32 v2, 31, v2
+; GFX11-NEXT: v_add_nc_u32_e32 v3, -1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_nc_u32_e32 v2, 32, v2
+; GFX11-NEXT: v_min_u32_e32 v2, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GFX11-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_sub_nc_u32_e32 v1, 32, v2
+; GFX11-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11FAKE16-LABEL: v_sitofp_i64_to_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_xor_b32_e32 v2, v0, v1
-; GFX11FAKE16-NEXT: v_cls_i32_e32 v3, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_ashrrev_i32_e32 v2, 31, v2
-; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v3, -1, v3
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add_nc_u32_e32 v2, 32, v2
-; GFX11FAKE16-NEXT: v_min_u32_e32 v2, v3, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
-; GFX11FAKE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11FAKE16-NEXT: v_sub_nc_u32_e32 v1, 32, v2
-; GFX11FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX1250TRUE16-LABEL: v_sitofp_i64_to_bf16:
+; GFX1250TRUE16: ; %bb.0:
+; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250TRUE16-NEXT: v_cls_i32_e32 v3, v1
+; GFX1250TRUE16-NEXT: v_xor_b32_e32 v2, v0, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_ashrrev_i32 v2, 31, v2
+; GFX1250TRUE16-NEXT: v_add_nc_u32_e32 v2, 32, v2
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_min_u32_e32 v2, v3, v2
+; GFX1250TRUE16-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1]
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX1250TRUE16-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX1250-LABEL: v_sitofp_i64_to_bf16:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_cls_i32_e32 v3, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_bitop2_b32 v2, v0, v1 bitop3:0x14
-; GFX1250-NEXT: v_ashrrev_i32_e32 v2, 31, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_nc_u32_e32 v2, 32, v2
-; GFX1250-NEXT: v_min_u32_e32 v2, v3, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1]
-; GFX1250-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
-; GFX1250-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250FAKE16-LABEL: v_sitofp_i64_to_bf16:
+; GFX1250FAKE16: ; %bb.0:
+; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250FAKE16-NEXT: v_cls_i32_e32 v3, v1
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_dual_add_nc_u32 v3, -1, v3 :: v_dual_bitop2_b32 v2, v0, v1 bitop3:0x14
+; GFX1250FAKE16-NEXT: v_ashrrev_i32_e32 v2, 31, v2
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_add_nc_u32_e32 v2, 32, v2
+; GFX1250FAKE16-NEXT: v_min_u32_e32 v2, v3, v2
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_lshlrev_b64_e32 v[0:1], v2, v[0:1]
+; GFX1250FAKE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_dual_sub_nc_u32 v1, 32, v2 :: v_dual_bitop2_b32 v0, v1, v0 bitop3:0x54
+; GFX1250FAKE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
%op = sitofp i64 %x to bfloat
ret bfloat %op
}
@@ -36499,10 +36160,10 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) #0 {
; GFX11TRUE16-LABEL: v_sitofp_v2i64_to_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v4, v0, v1
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v5, v2, v3
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v6, v1
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v7, v3
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v4, v2, v3
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v5, v0, v1
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v6, v3
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v7, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v4, 31, v4
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v5, 31, v5
@@ -36516,36 +36177,39 @@ define <2 x bfloat> @v_sitofp_v2i64_to_v2bf16(<2 x i64> %x) #0 {
; GFX11TRUE16-NEXT: v_min_u32_e32 v4, v6, v4
; GFX11TRUE16-NEXT: v_min_u32_e32 v5, v7, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v5, v[2:3]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v5, v[0:1]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v2, 32, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v4
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v2, v1
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v2i64_to_v2bf16:
@@ -36955,70 +36619,71 @@ define <3 x bfloat> @v_sitofp_v3i64_to_v3bf16(<3 x i64> %x) #0 {
; GFX11TRUE16-LABEL: v_sitofp_v3i64_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v7, v0, v1
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v8, v4, v5
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v9, v2, v3
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v10, v1
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v6, v5
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v7
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v8, 31, v8
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v11, v3
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v6, v2, v3
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v9, v0, v1
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v10, v3
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v7, v4, v5
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v11, v1
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v6, 31, v6
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v9, 31, v9
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v10, -1, v10
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v7, 32, v7
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v6, -1, v6
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, 32, v8
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v8, v5
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v7
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v6, 32, v6
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v11, -1, v11
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, 32, v9
-; GFX11TRUE16-NEXT: v_min_u32_e32 v7, v10, v7
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, -1, v8
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v7, 32, v7
+; GFX11TRUE16-NEXT: v_min_u32_e32 v6, v10, v6
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v6, v6, v8
-; GFX11TRUE16-NEXT: v_min_u32_e32 v8, v11, v9
+; GFX11TRUE16-NEXT: v_min_u32_e32 v9, v11, v9
+; GFX11TRUE16-NEXT: v_min_u32_e32 v7, v8, v7
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v6, v[2:3]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v9, v[0:1]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v8, v[2:3]
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v7, v[4:5]
; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v4
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v7
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v3, 1, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v6
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v9
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v7
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v4
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v2, v1
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v3i64_to_v3bf16:
@@ -37603,89 +37268,84 @@ define <4 x bfloat> @v_sitofp_v4i64_to_v4bf16(<4 x i64> %x) #0 {
; GFX11TRUE16-NEXT: v_xor_b32_e32 v8, v6, v7
; GFX11TRUE16-NEXT: v_cls_i32_e32 v9, v7
; GFX11TRUE16-NEXT: v_xor_b32_e32 v11, v4, v5
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v10, v5
; GFX11TRUE16-NEXT: v_xor_b32_e32 v13, v2, v3
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v10, v5
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v8, 31, v8
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, -1, v9
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v12, v3
; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v11, 31, v11
-; GFX11TRUE16-NEXT: v_xor_b32_e32 v15, v0, v1
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v14, v1
+; GFX11TRUE16-NEXT: v_xor_b32_e32 v14, v0, v1
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v8, 32, v8
-; GFX11TRUE16-NEXT: v_cls_i32_e32 v12, v3
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v10, -1, v10
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v12, -1, v12
; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v11, 32, v11
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v14, 31, v14
; GFX11TRUE16-NEXT: v_min_u32_e32 v8, v9, v8
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, -1, v10
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v10, 31, v13
-; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v13, 31, v15
+; GFX11TRUE16-NEXT: v_ashrrev_i32_e32 v9, 31, v13
+; GFX11TRUE16-NEXT: v_cls_i32_e32 v13, v1
+; GFX11TRUE16-NEXT: v_min_u32_e32 v10, v10, v11
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v14, 32, v14
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[6:7], v8, v[6:7]
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v9, v9, v11
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v10, 32, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v13, 32, v13
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v9, v[4:5]
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v9, 32, v9
+; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v13, -1, v13
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v10, v[4:5]
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v9, v12, v9
+; GFX11TRUE16-NEXT: v_min_u32_e32 v11, v13, v14
; GFX11TRUE16-NEXT: v_min_u32_e32 v6, 1, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v9, v[2:3]
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v11, v[0:1]
; GFX11TRUE16-NEXT: v_or_b32_e32 v6, v7, v6
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v7, 32, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v9
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v8, 32, v10
+; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v6, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v4, v4
-; GFX11TRUE16-NEXT: v_ldexp_f32 v4, v4, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v14, -1, v14
-; GFX11TRUE16-NEXT: v_min_u32_e32 v11, v14, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v11, v[0:1]
; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v6, v7
-; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_add_nc_u32_e32 v12, -1, v12
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v10, v12, v10
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v10, v[2:3]
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v10
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v11
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v3, v6, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v6, 32, v11
; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11TRUE16-NEXT: v_cvt_f32_i32_e32 v0, v0
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v8
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v9, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v6
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.h
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v7, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_sitofp_v4i64_to_v4bf16:
@@ -37909,7 +37569,7 @@ define bfloat @v_uitofp_i16_to_bf16(i16 %x) #0 {
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_i16_to_bf16:
@@ -38044,8 +37704,8 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v2i16_to_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
@@ -38061,8 +37721,11 @@ define <2 x bfloat> @v_uitofp_v2i16_to_v2bf16(<2 x i16> %x) #0 {
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v2i16_to_v2bf16:
@@ -38229,34 +37892,38 @@ define <3 x bfloat> @v_uitofp_v3i16_to_v3bf16(<3 x i16> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v3i16_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i16_to_v3bf16:
@@ -38482,39 +38149,47 @@ define <4 x bfloat> @v_uitofp_v4i16_to_v4bf16(<4 x i16> %x) #0 {
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v3, v3
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v11, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v9 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v4i16_to_v4bf16:
@@ -38640,33 +38315,19 @@ define bfloat @v_uitofp_i32_to_bf16(i32 %x) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_uitofp_i32_to_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_uitofp_i32_to_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_uitofp_i32_to_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_uitofp_i32_to_bf16:
; GFX1250: ; %bb.0:
@@ -38769,22 +38430,25 @@ define <2 x bfloat> @v_uitofp_v2i32_to_v2bf16(<2 x i32> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v2i32_to_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v2i32_to_v2bf16:
@@ -38939,29 +38603,32 @@ define <3 x bfloat> @v_uitofp_v3i32_to_v3bf16(<3 x i32> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v3i32_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i32_to_v3bf16:
@@ -39163,33 +38830,39 @@ define <4 x bfloat> @v_uitofp_v4i32_to_v4bf16(<4 x i32> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v4i32_to_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v3, v3
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v3, v3
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v9, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v6, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v4i32_to_v4bf16:
@@ -39344,53 +39017,29 @@ define bfloat @v_uitofp_i64_to_bf16(i64 %x) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_uitofp_i64_to_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v2, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 32, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_uitofp_i64_to_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_clz_i32_u32_e32 v2, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_min_u32_e32 v2, 32, v2
-; GFX11FAKE16-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11FAKE16-NEXT: v_sub_nc_u32_e32 v1, 32, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_uitofp_i64_to_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_clz_i32_u32_e32 v2, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_u32_e32 v2, 32, v2
+; GFX11-NEXT: v_lshlrev_b64 v[0:1], v2, v[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_sub_nc_u32_e32 v1, 32, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cvt_f32_u32_e32 v0, v0
+; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_uitofp_i64_to_bf16:
; GFX1250: ; %bb.0:
@@ -39586,42 +39235,45 @@ define <2 x bfloat> @v_uitofp_v2i64_to_v2bf16(<2 x i64> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v2i64_to_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v4, v1
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v5, v3
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v4, v3
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v5, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 32, v4
; GFX11TRUE16-NEXT: v_min_u32_e32 v5, 32, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v4, v[0:1]
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v5, v[2:3]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v4, v[2:3]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v5, v[0:1]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v2, 32, v4
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v4
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v3
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v2, v1
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v2i64_to_v2bf16:
@@ -39942,57 +39594,61 @@ define <3 x bfloat> @v_uitofp_v3i64_to_v3bf16(<3 x i64> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v3i64_to_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v6, v1
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v7, v5
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v8, v3
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v6, v3
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v7, v1
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v8, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v6, 32, v6
; GFX11TRUE16-NEXT: v_min_u32_e32 v7, 32, v7
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v8, 32, v8
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v6, v[0:1]
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v7, v[4:5]
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v8, v[2:3]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v6, v[2:3]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v7, v[0:1]
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v8, v[4:5]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, v5, v4
+; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v6
+; GFX11TRUE16-NEXT: v_min_u32_e32 v3, 1, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v1, 32, v6
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v7
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, v5, v3
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v1
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v8
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v4
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v2, v1
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
-; GFX11TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v6, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v3i64_to_v3bf16:
@@ -40445,68 +40101,75 @@ define <4 x bfloat> @v_uitofp_v4i64_to_v4bf16(<4 x i64> %x) #0 {
; GFX11TRUE16-LABEL: v_uitofp_v4i64_to_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v8, v7
; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v9, v5
; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v10, v3
; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v11, v1
-; GFX11TRUE16-NEXT: v_clz_i32_u32_e32 v8, v7
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v8, 32, v8
; GFX11TRUE16-NEXT: v_min_u32_e32 v9, 32, v9
-; GFX11TRUE16-NEXT: v_min_u32_e32 v10, 32, v10
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_min_u32_e32 v10, 32, v10
; GFX11TRUE16-NEXT: v_min_u32_e32 v11, 32, v11
-; GFX11TRUE16-NEXT: v_min_u32_e32 v8, 32, v8
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[6:7], v8, v[6:7]
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[4:5], v9, v[4:5]
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v10, v[2:3]
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshlrev_b64 v[2:3], v10, v[2:3]
; GFX11TRUE16-NEXT: v_lshlrev_b64 v[0:1], v11, v[0:1]
-; GFX11TRUE16-NEXT: v_lshlrev_b64 v[6:7], v8, v[6:7]
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v9, 32, v9
; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v8, 32, v8
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v9, 32, v9
+; GFX11TRUE16-NEXT: v_min_u32_e32 v6, 1, v6
; GFX11TRUE16-NEXT: v_min_u32_e32 v4, 1, v4
; GFX11TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
; GFX11TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11TRUE16-NEXT: v_min_u32_e32 v6, 1, v6
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, v7, v6
; GFX11TRUE16-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v2, v3, v2
; GFX11TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, v7, v6
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v5, 32, v10
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v3, v4
-; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v4, 32, v11
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v7, 32, v10
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v6
+; GFX11TRUE16-NEXT: v_sub_nc_u32_e32 v3, 32, v11
+; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v4, v4
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v2, v2
; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v0, v0
-; GFX11TRUE16-NEXT: v_cvt_f32_u32_e32 v1, v6
-; GFX11TRUE16-NEXT: v_ldexp_f32 v3, v3, v9
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v8
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v5
-; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v4
+; GFX11TRUE16-NEXT: v_ldexp_f32 v4, v4, v9
+; GFX11TRUE16-NEXT: v_ldexp_f32 v2, v2, v7
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v1, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v9, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
+; GFX11TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v10, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_uitofp_v4i64_to_v4bf16:
@@ -41081,11 +40744,13 @@ define <2 x bfloat> @v_vselect_v2bf16(<2 x i1> %cond, <2 x bfloat> %a, <2 x bflo
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.l
-; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo
; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v2.l, s0
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -41110,11 +40775,12 @@ define <2 x bfloat> @v_vselect_v2bf16(<2 x i1> %cond, <2 x bfloat> %a, <2 x bflo
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v1, 16, v2 :: v_dual_lshrrev_b32 v4, 16, v3
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.l
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, vcc_lo
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v2.l, s0
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -42827,19 +42493,21 @@ define <4 x bfloat> @v_vselect_v4bf16(<4 x i1> %cond, <4 x bfloat> %a, <4 x bflo
; GFX11TRUE16-LABEL: v_vselect_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
+; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
+; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v3.l
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.h, 1, v2.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v6
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v1.l
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v7
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v0.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v1.h
-; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v7.h, v5.h, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.h
+; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v6.h, v4.h, s0
+; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v1.l, s0
; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v4.l, s1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v5.l, s2
@@ -42873,19 +42541,20 @@ define <4 x bfloat> @v_vselect_v4bf16(<4 x i1> %cond, <4 x bfloat> %a, <4 x bflo
; GFX1250TRUE16: ; %bb.0:
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
-; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
+; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
+; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v3.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX1250TRUE16-NEXT: v_and_b16 v1.h, 1, v2.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v3, 16, v4 :: v_dual_lshrrev_b32 v8, 16, v6
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v1.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v1, 16, v5 :: v_dual_lshrrev_b32 v2, 16, v7
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v0.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v1.h
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v7.h, v5.h, vcc_lo
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.h
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v3.l, vcc_lo
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v6.h, v4.h, s0
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, v1.l, s0
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v4.l, s1
; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.l, v7.l, v5.l, s2
@@ -43159,27 +42828,35 @@ define <8 x bfloat> @v_vselect_v8bf16(<8 x i1> %cond, <8 x bfloat> %a, <8 x bflo
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v2.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.h, 1, v4.l
-; GFX11TRUE16-NEXT: v_and_b16 v2.l, 1, v6.l
+; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v4.l
+; GFX11TRUE16-NEXT: v_and_b16 v1.h, 1, v5.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v14
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v2.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.l
+; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v1.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v1.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v0.l
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v7.l
-; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v5.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v3.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v1.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v2.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v0.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v1.l
-; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v12.h, v8.h, s0
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.h
+; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v6.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v15
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v0.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v8
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v12
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v0.h
+; GFX11TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, v1.l, s5
+; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, v4.l, v3.l, s4
+; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v5.l, s2
+; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v16.l, v7.l, s0
; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, v12.l, v8.l, vcc_lo
-; GFX11TRUE16-NEXT: v_cndmask_b16 v3.h, v15.h, v11.h, s2
-; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, v14.h, v10.h, s3
-; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v13.h, v9.h, s4
; GFX11TRUE16-NEXT: v_cndmask_b16 v1.l, v13.l, v9.l, s1
-; GFX11TRUE16-NEXT: v_cndmask_b16 v2.l, v14.l, v10.l, s5
+; GFX11TRUE16-NEXT: v_cndmask_b16 v2.l, v14.l, v10.l, s3
; GFX11TRUE16-NEXT: v_cndmask_b16 v3.l, v15.l, v11.l, s6
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -43231,27 +42908,33 @@ define <8 x bfloat> @v_vselect_v8bf16(<8 x i1> %cond, <8 x bfloat> %a, <8 x bflo
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v2.l
-; GFX1250TRUE16-NEXT: v_and_b16 v1.h, 1, v4.l
-; GFX1250TRUE16-NEXT: v_and_b16 v2.l, 1, v6.l
+; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v4.l
+; GFX1250TRUE16-NEXT: v_and_b16 v1.h, 1, v5.l
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v9
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v2.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.l
+; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v1.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v1.h
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v0.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v7.l
-; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v5.l
-; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v3.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v1.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v2.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v0.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v1.l
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v12.h, v8.h, s0
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.h
+; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v6.l
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v1, 16, v11 :: v_dual_lshrrev_b32 v3, 16, v10
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v0.l
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v2, 16, v15 :: v_dual_lshrrev_b32 v4, 16, v14
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v6, 16, v13 :: v_dual_lshrrev_b32 v16, 16, v12
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v8
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v0.h
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, v1.l, s5
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, v4.l, v3.l, s4
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v5.l, s2
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v16.l, v7.l, s0
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, v12.l, v8.l, vcc_lo
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.h, v15.h, v11.h, s2
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, v14.h, v10.h, s3
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v13.h, v9.h, s4
; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.l, v13.l, v9.l, s1
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.l, v14.l, v10.l, s5
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.l, v14.l, v10.l, s3
; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.l, v15.l, v11.l, s6
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -43763,6 +43446,7 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: scratch_load_b32 v31, off, s32
+; GFX11TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v2.l
@@ -43776,9 +43460,24 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX11TRUE16-NEXT: v_and_b16 v5.l, 1, v10.l
; GFX11TRUE16-NEXT: v_and_b16 v5.h, 1, v11.l
; GFX11TRUE16-NEXT: v_and_b16 v6.l, 1, v12.l
-; GFX11TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
; GFX11TRUE16-NEXT: v_and_b16 v7.l, 1, v14.l
; GFX11TRUE16-NEXT: v_and_b16 v7.h, 1, v15.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v22
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v30
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v23
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v21
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v29
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v20
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v28
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v19
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v27
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v18
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v26
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v17
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v25
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v16
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v24
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.l
@@ -43790,28 +43489,29 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 1, v4.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 1, v4.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 1, v5.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 1, v6.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 1, v5.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 1, v7.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 1, v7.h
-; GFX11TRUE16-NEXT: v_cndmask_b16 v6.h, v30.h, v22.h, s10
+; GFX11TRUE16-NEXT: v_cndmask_b16 v6.h, v10.l, v9.l, s10
; GFX11TRUE16-NEXT: v_cndmask_b16 v6.l, v30.l, v22.l, s11
-; GFX11TRUE16-NEXT: v_cndmask_b16 v5.h, v29.h, v21.h, s12
+; GFX11TRUE16-NEXT: v_cndmask_b16 v5.h, v12.l, v11.l, s12
; GFX11TRUE16-NEXT: v_cndmask_b16 v5.l, v29.l, v21.l, s9
-; GFX11TRUE16-NEXT: v_cndmask_b16 v4.h, v28.h, v20.h, s8
+; GFX11TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v13.l, s8
; GFX11TRUE16-NEXT: v_cndmask_b16 v4.l, v28.l, v20.l, s7
-; GFX11TRUE16-NEXT: v_cndmask_b16 v3.h, v27.h, v19.h, s6
+; GFX11TRUE16-NEXT: v_cndmask_b16 v3.h, v32.l, v15.l, s6
; GFX11TRUE16-NEXT: v_cndmask_b16 v3.l, v27.l, v19.l, s5
-; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, v26.h, v18.h, s4
-; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v25.h, v17.h, s2
-; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v24.h, v16.h, s0
+; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, v34.l, v33.l, s4
+; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v36.l, v35.l, s2
+; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v38.l, v37.l, s0
; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, v24.l, v16.l, vcc_lo
; GFX11TRUE16-NEXT: v_cndmask_b16 v1.l, v25.l, v17.l, s1
; GFX11TRUE16-NEXT: v_cndmask_b16 v2.l, v26.l, v18.l, s3
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v7.h, v31.h, v23.h, s14
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v31
; GFX11TRUE16-NEXT: v_cndmask_b16 v7.l, v31.l, v23.l, s13
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b16 v7.h, v9.l, v8.l, s14
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_vselect_v16bf16:
@@ -43899,6 +43599,7 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250TRUE16-NEXT: scratch_load_b32 v31, off, s32
+; GFX1250TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v2.l
@@ -43912,9 +43613,17 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX1250TRUE16-NEXT: v_and_b16 v5.l, 1, v10.l
; GFX1250TRUE16-NEXT: v_and_b16 v5.h, 1, v11.l
; GFX1250TRUE16-NEXT: v_and_b16 v6.l, 1, v12.l
-; GFX1250TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
; GFX1250TRUE16-NEXT: v_and_b16 v7.l, 1, v14.l
; GFX1250TRUE16-NEXT: v_and_b16 v7.h, 1, v15.l
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v8, 16, v23 :: v_dual_lshrrev_b32 v9, 16, v22
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v10, 16, v30 :: v_dual_lshrrev_b32 v11, 16, v21
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.h
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v12, 16, v29 :: v_dual_lshrrev_b32 v13, 16, v20
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v14, 16, v28 :: v_dual_lshrrev_b32 v15, 16, v19
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v32, 16, v27 :: v_dual_lshrrev_b32 v33, 16, v18
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v34, 16, v26 :: v_dual_lshrrev_b32 v35, 16, v17
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v36, 16, v25 :: v_dual_lshrrev_b32 v37, 16, v16
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v24
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.l
@@ -43926,28 +43635,29 @@ define <16 x bfloat> @v_vselect_v16bf16(<16 x i1> %cond, <16 x bfloat> %a, <16 x
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 1, v4.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 1, v4.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 1, v5.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 1, v6.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 1, v5.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 1, v7.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 1, v7.h
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.h, v30.h, v22.h, s10
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.h, v10.l, v9.l, s10
; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.l, v30.l, v22.l, s11
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.h, v29.h, v21.h, s12
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.h, v12.l, v11.l, s12
; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.l, v29.l, v21.l, s9
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.h, v28.h, v20.h, s8
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v13.l, s8
; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.l, v28.l, v20.l, s7
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.h, v27.h, v19.h, s6
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.h, v32.l, v15.l, s6
; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.l, v27.l, v19.l, s5
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, v26.h, v18.h, s4
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v25.h, v17.h, s2
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v24.h, v16.h, s0
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, v34.l, v33.l, s4
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v36.l, v35.l, s2
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, v24.l, v16.l, vcc_lo
; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.l, v25.l, v17.l, s1
; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.l, v26.l, v18.l, s3
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.h, v31.h, v23.h, s14
+; GFX1250TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v31
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v38.l, v37.l, s0
; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.l, v31.l, v23.l, s13
+; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.h, v9.l, v8.l, s14
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250FAKE16-LABEL: v_vselect_v16bf16:
@@ -45287,153 +44997,200 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11TRUE16-NEXT: s_clause 0x1f
-; GFX11TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:64
-; GFX11TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:128
-; GFX11TRUE16-NEXT: scratch_load_b32 v34, off, s32 offset:60
-; GFX11TRUE16-NEXT: scratch_load_b32 v35, off, s32 offset:124
-; GFX11TRUE16-NEXT: scratch_load_b32 v36, off, s32 offset:56
-; GFX11TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:120
-; GFX11TRUE16-NEXT: scratch_load_b32 v38, off, s32 offset:52
-; GFX11TRUE16-NEXT: scratch_load_b32 v39, off, s32 offset:116
-; GFX11TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:48
-; GFX11TRUE16-NEXT: scratch_load_b32 v49, off, s32 offset:112
-; GFX11TRUE16-NEXT: scratch_load_b32 v50, off, s32 offset:44
-; GFX11TRUE16-NEXT: scratch_load_b32 v51, off, s32 offset:108
-; GFX11TRUE16-NEXT: scratch_load_b32 v52, off, s32 offset:40
-; GFX11TRUE16-NEXT: scratch_load_b32 v53, off, s32 offset:104
-; GFX11TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:36
-; GFX11TRUE16-NEXT: scratch_load_b32 v55, off, s32 offset:100
-; GFX11TRUE16-NEXT: scratch_load_b32 v64, off, s32 offset:32
-; GFX11TRUE16-NEXT: scratch_load_b32 v65, off, s32 offset:96
-; GFX11TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:28
-; GFX11TRUE16-NEXT: scratch_load_b32 v67, off, s32 offset:92
-; GFX11TRUE16-NEXT: scratch_load_b32 v68, off, s32 offset:24
-; GFX11TRUE16-NEXT: scratch_load_b32 v69, off, s32 offset:88
-; GFX11TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:20
-; GFX11TRUE16-NEXT: scratch_load_b32 v71, off, s32 offset:84
-; GFX11TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:16
-; GFX11TRUE16-NEXT: scratch_load_b32 v81, off, s32 offset:80
-; GFX11TRUE16-NEXT: scratch_load_b32 v82, off, s32 offset:12
-; GFX11TRUE16-NEXT: scratch_load_b32 v83, off, s32 offset:76
-; GFX11TRUE16-NEXT: scratch_load_b32 v84, off, s32 offset:8
-; GFX11TRUE16-NEXT: scratch_load_b32 v85, off, s32 offset:72
-; GFX11TRUE16-NEXT: scratch_load_b32 v86, off, s32 offset:4
-; GFX11TRUE16-NEXT: scratch_load_b32 v87, off, s32 offset:68
+; GFX11TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32
+; GFX11TRUE16-NEXT: scratch_load_b32 v31, off, s32 offset:64
+; GFX11TRUE16-NEXT: scratch_load_b32 v32, off, s32 offset:128
+; GFX11TRUE16-NEXT: scratch_load_b32 v33, off, s32 offset:60
+; GFX11TRUE16-NEXT: scratch_load_b32 v34, off, s32 offset:124
+; GFX11TRUE16-NEXT: scratch_load_b32 v35, off, s32 offset:56
+; GFX11TRUE16-NEXT: scratch_load_b32 v36, off, s32 offset:120
+; GFX11TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:52
+; GFX11TRUE16-NEXT: scratch_load_b32 v38, off, s32 offset:116
+; GFX11TRUE16-NEXT: scratch_load_b32 v39, off, s32 offset:48
+; GFX11TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:112
+; GFX11TRUE16-NEXT: scratch_load_b32 v49, off, s32 offset:44
+; GFX11TRUE16-NEXT: scratch_load_b32 v50, off, s32 offset:108
+; GFX11TRUE16-NEXT: scratch_load_b32 v51, off, s32 offset:40
+; GFX11TRUE16-NEXT: scratch_load_b32 v52, off, s32 offset:104
+; GFX11TRUE16-NEXT: scratch_load_b32 v53, off, s32 offset:36
+; GFX11TRUE16-NEXT: scratch_load_b32 v54, off, s32 offset:100
+; GFX11TRUE16-NEXT: scratch_load_b32 v55, off, s32 offset:32
+; GFX11TRUE16-NEXT: scratch_load_b32 v64, off, s32 offset:96
+; GFX11TRUE16-NEXT: scratch_load_b32 v65, off, s32 offset:28
+; GFX11TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:92
+; GFX11TRUE16-NEXT: scratch_load_b32 v67, off, s32 offset:24
+; GFX11TRUE16-NEXT: scratch_load_b32 v68, off, s32 offset:88
+; GFX11TRUE16-NEXT: scratch_load_b32 v69, off, s32 offset:20
+; GFX11TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:84
+; GFX11TRUE16-NEXT: scratch_load_b32 v71, off, s32 offset:16
+; GFX11TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:80
+; GFX11TRUE16-NEXT: scratch_load_b32 v81, off, s32 offset:12
+; GFX11TRUE16-NEXT: scratch_load_b32 v82, off, s32 offset:76
+; GFX11TRUE16-NEXT: scratch_load_b32 v83, off, s32 offset:8
+; GFX11TRUE16-NEXT: scratch_load_b32 v84, off, s32 offset:72
+; GFX11TRUE16-NEXT: scratch_load_b32 v85, off, s32 offset:4
+; GFX11TRUE16-NEXT: scratch_load_b32 v86, off, s32 offset:68
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(32)
; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v2.l
-; GFX11TRUE16-NEXT: v_and_b16 v1.h, 1, v3.l
-; GFX11TRUE16-NEXT: v_and_b16 v2.l, 1, v4.l
-; GFX11TRUE16-NEXT: v_and_b16 v2.h, 1, v5.l
-; GFX11TRUE16-NEXT: v_and_b16 v3.l, 1, v6.l
-; GFX11TRUE16-NEXT: v_and_b16 v3.h, 1, v7.l
-; GFX11TRUE16-NEXT: v_and_b16 v4.l, 1, v8.l
-; GFX11TRUE16-NEXT: v_and_b16 v4.h, 1, v9.l
-; GFX11TRUE16-NEXT: v_and_b16 v5.l, 1, v10.l
-; GFX11TRUE16-NEXT: v_and_b16 v5.h, 1, v11.l
-; GFX11TRUE16-NEXT: v_and_b16 v6.l, 1, v12.l
-; GFX11TRUE16-NEXT: v_and_b16 v6.h, 1, v13.l
-; GFX11TRUE16-NEXT: v_and_b16 v7.l, 1, v14.l
-; GFX11TRUE16-NEXT: v_and_b16 v7.h, 1, v15.l
-; GFX11TRUE16-NEXT: v_and_b16 v8.l, 1, v16.l
-; GFX11TRUE16-NEXT: v_and_b16 v8.h, 1, v17.l
-; GFX11TRUE16-NEXT: v_and_b16 v9.l, 1, v18.l
-; GFX11TRUE16-NEXT: v_and_b16 v9.h, 1, v19.l
-; GFX11TRUE16-NEXT: v_and_b16 v10.l, 1, v20.l
-; GFX11TRUE16-NEXT: v_and_b16 v10.h, 1, v21.l
-; GFX11TRUE16-NEXT: v_and_b16 v11.l, 1, v22.l
-; GFX11TRUE16-NEXT: v_and_b16 v11.h, 1, v23.l
-; GFX11TRUE16-NEXT: v_and_b16 v12.l, 1, v24.l
-; GFX11TRUE16-NEXT: v_and_b16 v12.h, 1, v25.l
-; GFX11TRUE16-NEXT: v_and_b16 v13.l, 1, v26.l
-; GFX11TRUE16-NEXT: v_and_b16 v13.h, 1, v27.l
-; GFX11TRUE16-NEXT: v_and_b16 v14.l, 1, v28.l
-; GFX11TRUE16-NEXT: v_and_b16 v14.h, 1, v29.l
-; GFX11TRUE16-NEXT: v_and_b16 v15.l, 1, v30.l
+; GFX11TRUE16-NEXT: v_and_b16 v7.h, 1, v14.l
+; GFX11TRUE16-NEXT: v_and_b16 v8.h, 1, v16.l
+; GFX11TRUE16-NEXT: v_and_b16 v9.h, 1, v18.l
+; GFX11TRUE16-NEXT: v_and_b16 v10.h, 1, v20.l
+; GFX11TRUE16-NEXT: v_and_b16 v11.h, 1, v22.l
+; GFX11TRUE16-NEXT: v_and_b16 v12.h, 1, v24.l
+; GFX11TRUE16-NEXT: v_and_b16 v13.h, 1, v26.l
+; GFX11TRUE16-NEXT: v_and_b16 v14.h, 1, v28.l
+; GFX11TRUE16-NEXT: v_and_b16 v15.h, 1, v30.l
+; GFX11TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
+; GFX11TRUE16-NEXT: v_and_b16 v1.h, 1, v2.l
+; GFX11TRUE16-NEXT: v_and_b16 v2.l, 1, v3.l
+; GFX11TRUE16-NEXT: v_and_b16 v2.h, 1, v4.l
+; GFX11TRUE16-NEXT: v_and_b16 v3.l, 1, v5.l
+; GFX11TRUE16-NEXT: v_and_b16 v3.h, 1, v6.l
+; GFX11TRUE16-NEXT: v_and_b16 v4.l, 1, v7.l
+; GFX11TRUE16-NEXT: v_and_b16 v4.h, 1, v8.l
+; GFX11TRUE16-NEXT: v_and_b16 v5.l, 1, v9.l
+; GFX11TRUE16-NEXT: v_and_b16 v5.h, 1, v10.l
+; GFX11TRUE16-NEXT: v_and_b16 v6.l, 1, v11.l
+; GFX11TRUE16-NEXT: v_and_b16 v6.h, 1, v12.l
+; GFX11TRUE16-NEXT: v_and_b16 v7.l, 1, v13.l
+; GFX11TRUE16-NEXT: v_and_b16 v8.l, 1, v15.l
+; GFX11TRUE16-NEXT: v_and_b16 v9.l, 1, v17.l
+; GFX11TRUE16-NEXT: v_and_b16 v10.l, 1, v19.l
+; GFX11TRUE16-NEXT: v_and_b16 v11.l, 1, v21.l
+; GFX11TRUE16-NEXT: v_and_b16 v12.l, 1, v23.l
+; GFX11TRUE16-NEXT: v_and_b16 v13.l, 1, v25.l
+; GFX11TRUE16-NEXT: v_and_b16 v14.l, 1, v27.l
+; GFX11TRUE16-NEXT: v_and_b16 v15.l, 1, v29.l
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v1.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v2.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v2.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v3.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v3.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 1, v4.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 1, v4.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 1, v5.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v5.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 1, v6.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 1, v6.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 1, v7.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 1, v7.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 1, v8.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 1, v8.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 1, v9.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 1, v9.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 1, v10.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 1, v10.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 1, v11.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 1, v11.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 1, v12.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 1, v12.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 1, v13.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 1, v15.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 1, v7.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 1, v8.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 1, v9.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 1, v10.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 1, v11.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 1, v12.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 1, v13.h
; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 1, v14.h
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 1, v14.l
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 1, v13.h
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(32)
-; GFX11TRUE16-NEXT: v_and_b16 v0.l, 1, v31.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 1, v15.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v1.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v1.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v2.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v2.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v3.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v3.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v4.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 1, v4.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 1, v5.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 1, v5.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 1, v6.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 1, v6.h
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 1, v7.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 1, v8.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 1, v9.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 1, v10.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 1, v11.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 1, v12.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 1, v13.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 1, v14.l
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 1, v15.l
+; GFX11TRUE16-NEXT: v_and_b16 v0.h, 1, v0.h
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(31)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v31
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(30)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v32.l, s26
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v32
+; GFX11TRUE16-NEXT: v_cndmask_b16 v15.l, v32.l, v31.l, s28
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(29)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v33
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(28)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v14.h, v35.h, v34.h, s27
-; GFX11TRUE16-NEXT: v_cndmask_b16 v14.l, v35.l, v34.l, s28
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v34
+; GFX11TRUE16-NEXT: v_cndmask_b16 v14.l, v34.l, v33.l, s27
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(27)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v35
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(26)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v13.h, v37.h, v36.h, s29
-; GFX11TRUE16-NEXT: v_cndmask_b16 v13.l, v37.l, v36.l, s25
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v36
+; GFX11TRUE16-NEXT: v_cndmask_b16 v13.l, v36.l, v35.l, s25
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(25)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v37
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(24)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v12.h, v39.h, v38.h, s24
-; GFX11TRUE16-NEXT: v_cndmask_b16 v12.l, v39.l, v38.l, s23
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v38
+; GFX11TRUE16-NEXT: v_cndmask_b16 v12.l, v38.l, v37.l, s23
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(23)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v39
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(22)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v11.h, v49.h, v48.h, s22
-; GFX11TRUE16-NEXT: v_cndmask_b16 v11.l, v49.l, v48.l, s21
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v48
+; GFX11TRUE16-NEXT: v_cndmask_b16 v11.l, v48.l, v39.l, s21
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(21)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v49
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(20)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v10.h, v51.h, v50.h, s20
-; GFX11TRUE16-NEXT: v_cndmask_b16 v10.l, v51.l, v50.l, s19
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v50
+; GFX11TRUE16-NEXT: v_cndmask_b16 v10.l, v50.l, v49.l, s19
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(19)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v51
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(18)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v9.h, v53.h, v52.h, s18
-; GFX11TRUE16-NEXT: v_cndmask_b16 v9.l, v53.l, v52.l, s17
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v29, 16, v52
+; GFX11TRUE16-NEXT: v_cndmask_b16 v9.l, v52.l, v51.l, s17
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(17)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v53
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(16)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v8.h, v55.h, v54.h, s16
-; GFX11TRUE16-NEXT: v_cndmask_b16 v8.l, v55.l, v54.l, s15
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v31, 16, v54
+; GFX11TRUE16-NEXT: v_cndmask_b16 v8.l, v54.l, v53.l, s15
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(15)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v55
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(14)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v7.h, v65.h, v64.h, s14
-; GFX11TRUE16-NEXT: v_cndmask_b16 v7.l, v65.l, v64.l, s13
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v64
+; GFX11TRUE16-NEXT: v_cndmask_b16 v7.l, v64.l, v55.l, s13
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(13)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v65
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(12)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v6.h, v67.h, v66.h, s12
-; GFX11TRUE16-NEXT: v_cndmask_b16 v6.l, v67.l, v66.l, s11
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v66
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(11)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v67
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(10)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v5.h, v69.h, v68.h, s10
-; GFX11TRUE16-NEXT: v_cndmask_b16 v5.l, v69.l, v68.l, s9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v68
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(9)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v69
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(8)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v4.h, v71.h, v70.h, s8
-; GFX11TRUE16-NEXT: v_cndmask_b16 v4.l, v71.l, v70.l, s7
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v70
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(7)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v71
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v3.h, v81.h, v80.h, s6
-; GFX11TRUE16-NEXT: v_cndmask_b16 v3.l, v81.l, v80.l, s5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v80
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(5)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v50, 16, v81
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(4)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, v83.h, v82.h, s4
-; GFX11TRUE16-NEXT: v_cndmask_b16 v2.l, v83.l, v82.l, s3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v82
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v83
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v85.h, v84.h, s2
-; GFX11TRUE16-NEXT: v_cndmask_b16 v1.l, v85.l, v84.l, s1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v53, 16, v84
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v85
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v87.h, v86.h, s0
-; GFX11TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.l
-; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, v87.l, v86.l, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b16 v15.h, v33.h, v32.h, s0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v55, 16, v86
+; GFX11TRUE16-NEXT: v_cndmask_b16 v0.l, v86.l, v85.l, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX11TRUE16-NEXT: v_cndmask_b16 v6.l, v66.l, v65.l, s11
+; GFX11TRUE16-NEXT: v_cndmask_b16 v5.l, v68.l, v67.l, s9
+; GFX11TRUE16-NEXT: v_cndmask_b16 v4.l, v70.l, v69.l, s7
+; GFX11TRUE16-NEXT: v_cndmask_b16 v3.l, v80.l, v71.l, s5
+; GFX11TRUE16-NEXT: v_cndmask_b16 v2.l, v82.l, v81.l, s3
+; GFX11TRUE16-NEXT: v_cndmask_b16 v1.l, v84.l, v83.l, s1
+; GFX11TRUE16-NEXT: v_cndmask_b16 v14.h, v19.l, v18.l, s29
+; GFX11TRUE16-NEXT: v_cndmask_b16 v13.h, v21.l, v20.l, s26
+; GFX11TRUE16-NEXT: v_cndmask_b16 v12.h, v23.l, v22.l, s24
+; GFX11TRUE16-NEXT: v_cndmask_b16 v11.h, v25.l, v24.l, s22
+; GFX11TRUE16-NEXT: v_cndmask_b16 v10.h, v27.l, v26.l, s20
+; GFX11TRUE16-NEXT: v_cndmask_b16 v9.h, v29.l, v28.l, s18
+; GFX11TRUE16-NEXT: v_cndmask_b16 v8.h, v31.l, v30.l, s16
+; GFX11TRUE16-NEXT: v_cndmask_b16 v7.h, v33.l, v32.l, s14
+; GFX11TRUE16-NEXT: v_cndmask_b16 v6.h, v35.l, v34.l, s12
+; GFX11TRUE16-NEXT: v_cndmask_b16 v5.h, v37.l, v36.l, s10
+; GFX11TRUE16-NEXT: v_cndmask_b16 v4.h, v39.l, v38.l, s8
+; GFX11TRUE16-NEXT: v_cndmask_b16 v0.h, v55.l, v54.l, s0
+; GFX11TRUE16-NEXT: v_cndmask_b16 v1.h, v53.l, v52.l, s2
+; GFX11TRUE16-NEXT: v_cndmask_b16 v2.h, v51.l, v50.l, s4
+; GFX11TRUE16-NEXT: v_cndmask_b16 v3.h, v49.l, v48.l, s6
+; GFX11TRUE16-NEXT: v_cndmask_b16 v15.h, v17.l, v16.l, vcc_lo
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_vselect_v32bf16:
@@ -45675,41 +45432,49 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GFX1250TRUE16-NEXT: scratch_load_b32 v87, off, s32 offset:68
; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v2.l
-; GFX1250TRUE16-NEXT: v_and_b16 v1.h, 1, v3.l
-; GFX1250TRUE16-NEXT: v_and_b16 v2.l, 1, v8.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v4.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v0.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v1.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v1.h
-; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v5.l
; GFX1250TRUE16-NEXT: v_and_b16 v1.l, 1, v6.l
; GFX1250TRUE16-NEXT: v_and_b16 v1.h, 1, v7.l
; GFX1250TRUE16-NEXT: v_and_b16 v2.h, 1, v9.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
+; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v2.l
+; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
+; GFX1250TRUE16-NEXT: v_and_b16 v2.l, 1, v8.l
; GFX1250TRUE16-NEXT: v_and_b16 v3.l, 1, v10.l
; GFX1250TRUE16-NEXT: v_and_b16 v3.h, 1, v11.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v0.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 1, v0.h
+; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v4.l
+; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v5.l
; GFX1250TRUE16-NEXT: v_and_b16 v4.l, 1, v12.l
; GFX1250TRUE16-NEXT: v_and_b16 v4.h, 1, v13.l
+; GFX1250TRUE16-NEXT: v_and_b16 v6.l, 1, v16.l
+; GFX1250TRUE16-NEXT: v_and_b16 v7.l, 1, v18.l
+; GFX1250TRUE16-NEXT: v_and_b16 v8.l, 1, v20.l
+; GFX1250TRUE16-NEXT: v_and_b16 v9.l, 1, v22.l
+; GFX1250TRUE16-NEXT: v_and_b16 v10.l, 1, v24.l
+; GFX1250TRUE16-NEXT: v_and_b16 v11.l, 1, v26.l
+; GFX1250TRUE16-NEXT: v_and_b16 v12.l, 1, v28.l
+; GFX1250TRUE16-NEXT: v_and_b16 v13.l, 1, v30.l
; GFX1250TRUE16-NEXT: v_and_b16 v5.l, 1, v14.l
; GFX1250TRUE16-NEXT: v_and_b16 v5.h, 1, v15.l
-; GFX1250TRUE16-NEXT: v_and_b16 v6.l, 1, v16.l
; GFX1250TRUE16-NEXT: v_and_b16 v6.h, 1, v17.l
-; GFX1250TRUE16-NEXT: v_and_b16 v7.l, 1, v18.l
; GFX1250TRUE16-NEXT: v_and_b16 v7.h, 1, v19.l
-; GFX1250TRUE16-NEXT: v_and_b16 v8.l, 1, v20.l
; GFX1250TRUE16-NEXT: v_and_b16 v8.h, 1, v21.l
-; GFX1250TRUE16-NEXT: v_and_b16 v9.l, 1, v22.l
; GFX1250TRUE16-NEXT: v_and_b16 v9.h, 1, v23.l
-; GFX1250TRUE16-NEXT: v_and_b16 v10.l, 1, v24.l
; GFX1250TRUE16-NEXT: v_and_b16 v10.h, 1, v25.l
-; GFX1250TRUE16-NEXT: v_and_b16 v11.l, 1, v26.l
; GFX1250TRUE16-NEXT: v_and_b16 v11.h, 1, v27.l
-; GFX1250TRUE16-NEXT: v_and_b16 v12.l, 1, v28.l
; GFX1250TRUE16-NEXT: v_and_b16 v12.h, 1, v29.l
-; GFX1250TRUE16-NEXT: v_and_b16 v13.l, 1, v30.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v0.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 1, v0.h
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 1, v6.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 1, v7.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 1, v8.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 1, v9.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 1, v10.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 1, v11.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 1, v13.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 1, v12.l
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 1, v0.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 1, v1.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 1, v1.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 1, v2.l
@@ -45720,73 +45485,80 @@ define <32 x bfloat> @v_vselect_v32bf16(<32 x i1> %cond, <32 x bfloat> %a, <32 x
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 1, v4.h
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 1, v5.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 1, v5.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 1, v6.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 1, v6.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 1, v7.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 1, v7.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 1, v8.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s20, 1, v8.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s21, 1, v9.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s22, 1, v9.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s23, 1, v10.l
; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s24, 1, v10.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s25, 1, v11.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s26, 1, v13.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 1, v12.h
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s28, 1, v12.l
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 1, v11.h
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s27, 1, v11.h
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s29, 1, v12.h
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x20
-; GFX1250TRUE16-NEXT: v_and_b16 v0.l, 1, v31.l
+; GFX1250TRUE16-NEXT: v_and_b16 v0.h, 1, v31.l
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x1e
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v16, 16, v32 :: v_dual_lshrrev_b32 v17, 16, v33
; GFX1250TRUE16-NEXT: v_cndmask_b16 v15.l, v33.l, v32.l, s26
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x1c
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v14.h, v35.h, v34.h, s27
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v18, 16, v34 :: v_dual_lshrrev_b32 v19, 16, v35
; GFX1250TRUE16-NEXT: v_cndmask_b16 v14.l, v35.l, v34.l, s28
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x1a
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v13.h, v37.h, v36.h, s29
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v20, 16, v36 :: v_dual_lshrrev_b32 v21, 16, v37
; GFX1250TRUE16-NEXT: v_cndmask_b16 v13.l, v37.l, v36.l, s25
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x18
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v12.h, v39.h, v38.h, s24
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v22, 16, v38 :: v_dual_lshrrev_b32 v23, 16, v39
; GFX1250TRUE16-NEXT: v_cndmask_b16 v12.l, v39.l, v38.l, s23
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x16
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v11.h, v49.h, v48.h, s22
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v24, 16, v48 :: v_dual_lshrrev_b32 v25, 16, v49
; GFX1250TRUE16-NEXT: v_cndmask_b16 v11.l, v49.l, v48.l, s21
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x14
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v10.h, v51.h, v50.h, s20
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v26, 16, v50 :: v_dual_lshrrev_b32 v27, 16, v51
; GFX1250TRUE16-NEXT: v_cndmask_b16 v10.l, v51.l, v50.l, s19
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x12
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v9.h, v53.h, v52.h, s18
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v28, 16, v52 :: v_dual_lshrrev_b32 v29, 16, v53
; GFX1250TRUE16-NEXT: v_cndmask_b16 v9.l, v53.l, v52.l, s17
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x10
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v8.h, v55.h, v54.h, s16
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v30, 16, v54 :: v_dual_lshrrev_b32 v31, 16, v55
; GFX1250TRUE16-NEXT: v_cndmask_b16 v8.l, v55.l, v54.l, s15
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0xe
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.h, v65.h, v64.h, s14
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.l, v65.l, v64.l, s13
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v32, 16, v64 :: v_dual_lshrrev_b32 v33, 16, v65
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0xc
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.h, v67.h, v66.h, s12
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.l, v67.l, v66.l, s11
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v34, 16, v66 :: v_dual_lshrrev_b32 v35, 16, v67
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0xa
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.h, v69.h, v68.h, s10
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.l, v69.l, v68.l, s9
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v36, 16, v68 :: v_dual_lshrrev_b32 v37, 16, v69
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x8
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.h, v71.h, v70.h, s8
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.l, v71.l, v70.l, s7
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v38, 16, v70 :: v_dual_lshrrev_b32 v39, 16, v71
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x6
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.h, v81.h, v80.h, s6
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.l, v81.l, v80.l, s5
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v48, 16, v80 :: v_dual_lshrrev_b32 v49, 16, v81
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x4
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, v83.h, v82.h, s4
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.l, v83.l, v82.l, s3
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v50, 16, v82 :: v_dual_lshrrev_b32 v51, 16, v83
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x2
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v85.h, v84.h, s2
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.l, v85.l, v84.l, s0
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v52, 16, v84 :: v_dual_lshrrev_b32 v53, 16, v85
; GFX1250TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v87.h, v86.h, s1
-; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 1, v0.l
+; GFX1250TRUE16-NEXT: v_dual_lshrrev_b32 v54, 16, v86 :: v_dual_lshrrev_b32 v55, 16, v87
; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.l, v87.l, v86.l, vcc_lo
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250TRUE16-NEXT: v_cndmask_b16 v15.h, v33.h, v32.h, s1
+; GFX1250TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.l, v65.l, v64.l, s13
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.l, v67.l, v66.l, s11
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.l, v69.l, v68.l, s9
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.l, v71.l, v70.l, s7
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.l, v81.l, v80.l, s5
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.l, v83.l, v82.l, s3
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.l, v85.l, v84.l, s1
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v14.h, v19.l, v18.l, s29
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v13.h, v21.l, v20.l, s27
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v12.h, v23.l, v22.l, s24
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v11.h, v25.l, v24.l, s22
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v10.h, v27.l, v26.l, s20
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v9.h, v29.l, v28.l, s18
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v8.h, v31.l, v30.l, s16
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v7.h, v33.l, v32.l, s14
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v6.h, v35.l, v34.l, s12
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v5.h, v37.l, v36.l, s10
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v4.h, v39.l, v38.l, s8
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v0.h, v55.l, v54.l, s0
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v1.h, v53.l, v52.l, s2
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v2.h, v51.l, v50.l, s4
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v3.h, v49.l, v48.l, s6
+; GFX1250TRUE16-NEXT: v_cndmask_b16 v15.h, v17.l, v16.l, vcc_lo
; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250FAKE16-LABEL: v_vselect_v32bf16:
@@ -46055,41 +45827,23 @@ define bfloat @v_fma_bf16(bfloat %a, bfloat %b, bfloat %c) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fma_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v2, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fma_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v0, v2, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_fma_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX11-NEXT: v_bfe_u32 v0, v2, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_fma_bf16:
; GFX1250: ; %bb.0:
@@ -46226,12 +45980,12 @@ define <2 x bfloat> @v_fma_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; GFX11TRUE16-LABEL: v_fma_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v2, v0, v1 :: v_dual_fmac_f32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
@@ -46242,12 +45996,14 @@ define <2 x bfloat> @v_fma_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat>
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v4, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v2bf16:
@@ -46459,40 +46215,43 @@ define <3 x bfloat> @v_fma_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfloat>
; GFX11TRUE16-LABEL: v_fma_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v5, v1, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v6, v8, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v0, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v0, 0x400000, v6
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v7, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v6, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v3bf16:
@@ -46760,48 +46519,52 @@ define <4 x bfloat> @v_fma_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfloat>
; GFX11TRUE16-LABEL: v_fma_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v7, 16, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_and_b32 v7, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v9, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v7, v10, v8
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v9, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v7
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v7, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v9, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v0, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v4bf16:
@@ -47256,92 +47019,93 @@ define <8 x bfloat> @v_fma_v8bf16(<8 x bfloat> %a, <8 x bfloat> %b, <8 x bfloat>
; GFX11TRUE16-LABEL: v_fma_v8bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v11
; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v7
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v11
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v11, v3, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v10
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v6
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v3, v15, v7 :: v_dual_lshlrev_b32 v6, 16, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v10
+; GFX11TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v7, v2, v6
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v12, v14, v13
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v11
; GFX11TRUE16-NEXT: v_bfe_u32 v13, v12, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v12
+; GFX11TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v12
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_add3_u32 v13, v13, v12, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v13, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_bfe_u32 v10, v11, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v13, v14, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v14, v3, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v7, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v14, v16, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v10, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v11
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v14, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11TRUE16-NEXT: v_bfe_u32 v15, v14, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v2, v6, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v15, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v9
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v9
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v14
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.h
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v9, v1, v5
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v12, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v12.l
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v10, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v11, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v4, v10, v11 :: v_dual_and_b32 v5, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_add3_u32 v7, v7, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v15, v17, v16
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_bfe_u32 v12, v15, 16, 1
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v13, v16, v14
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v0, v1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v15, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v13, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v13
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v13, v15, v14 :: v_dual_and_b32 v14, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v9, v1, v5 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v10, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v7, v13, 16, 1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v13
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v10, v14, v11
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v7, v13, 0x7fff
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v13, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v9, v11, v5, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v0, v12, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.h
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v8, v0, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v9, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v4, v10, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v10
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v10, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v9, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
+; GFX11TRUE16-NEXT: v_add3_u32 v5, v5, v8, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v13, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v7.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v8bf16:
@@ -48201,153 +47965,166 @@ define <16 x bfloat> @v_fma_v16bf16(<16 x bfloat> %a, <16 x bfloat> %b, <16 x bf
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v23
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v24, v26, v25 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
; GFX11TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v22
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
; GFX11TRUE16-NEXT: v_bfe_u32 v25, v24, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v26, v28, v27
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v22, v6, v14
-; GFX11TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v28, 0xffff0000, v13
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11TRUE16-NEXT: v_add3_u32 v25, v25, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v29, 0x400000, v24
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v22
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v26, v28, v27 :: v_dual_and_b32 v27, 0xffff0000, v5
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v22, v6, v14 :: v_dual_lshlrev_b32 v15, 16, v15
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v23, v7, v15
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v25, v29, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v15, v23, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v24, v26, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v23
+; GFX11TRUE16-NEXT: v_add3_u32 v7, v25, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v25, v23, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v23
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v26
-; GFX11TRUE16-NEXT: v_add3_u32 v15, v15, v23, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v24, v24, v26, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v15, v25, v23, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v13
+; GFX11TRUE16-NEXT: v_bfe_u32 v23, v26, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v7
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v15, v15, v24 :: v_dual_and_b32 v24, 0xffff0000, v21
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v23, v26, 0x7fff
; GFX11TRUE16-NEXT: v_bfe_u32 v23, v22, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v14, v29, v28 :: v_dual_cndmask_b32 v15, v15, v25
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
-; GFX11TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v24, v27, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v20
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
-; GFX11TRUE16-NEXT: v_add3_u32 v23, v23, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v22
-; GFX11TRUE16-NEXT: v_bfe_u32 v28, v14, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v15
+; GFX11TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v26
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v24, v27, v25
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v14.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v15, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v15, v23, v22, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v22
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v20, v4, v12
-; GFX11TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v19
-; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v24, v26, v25
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v14
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v21, v5, v13
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v23, v27, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v28, v14, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11TRUE16-NEXT: v_bfe_u32 v25, v24, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v21, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v27, v20, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v24
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v22, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v6
+; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v20
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v6, v15, v23 :: v_dual_and_b32 v13, 0xffff0000, v12
+; GFX11TRUE16-NEXT: v_bfe_u32 v15, v24, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v24
+; GFX11TRUE16-NEXT: v_bfe_u32 v26, v21, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11TRUE16-NEXT: v_add3_u32 v15, v15, v24, 0x7fff
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v22, v13
; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v21
-; GFX11TRUE16-NEXT: v_add3_u32 v14, v23, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v13, v15, v23 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_add3_u32 v15, v26, v21, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v23, v5, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: v_add3_u32 v23, v25, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v21, v27, v20, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v14, v22, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v20
-; GFX11TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v18
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v14.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v12, v25, v4
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v23, v26, vcc_lo
-; GFX11TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v10
-; GFX11TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v25.l
+; GFX11TRUE16-NEXT: v_add3_u32 v21, v23, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v15, v22, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v5
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v15
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v21, v21, v22 :: v_dual_fmac_f32 v20, v4, v12
+; GFX11TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v19
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v12, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v22, v20, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v20
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v12, v23, v4 :: v_dual_lshlrev_b32 v11, 16, v11
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v22, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v13.l
+; GFX11TRUE16-NEXT: v_bfe_u32 v22, v12, 16, 1
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v19, v3, v11
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v4, v4, v21 :: v_dual_and_b32 v3, 0xffff0000, v18
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v12
+; GFX11TRUE16-NEXT: v_add3_u32 v20, v22, v12, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v10
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v24, v26, v25
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v20, v21, v22 :: v_dual_and_b32 v25, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_add3_u32 v21, v23, v12, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v24, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v12
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.h
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v23, v24, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v19, v3, v11
-; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v3, v21, v22 :: v_dual_and_b32 v22, 0xffff0000, v17
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v3, v22, v11 :: v_dual_cndmask_b32 v12, v20, v21
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v19, 16, 1
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v18
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v19, 16, 1
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v22, v25, v23 :: v_dual_fmac_f32 v11, v2, v10
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v19
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v18, v19, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v24
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v8
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v11, 16, 1
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v17, v1, v9 :: v_dual_cndmask_b32 v10, v2, v10
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v16
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v8
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v12, v18, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v21, v11, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v11
+; GFX11TRUE16-NEXT: v_bfe_u32 v21, v3, 16, 1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v20, v19, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v19
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v11, v2, v10
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v21, v3, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v9
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v2, v18, v20 :: v_dual_lshlrev_b32 v9, 16, v9
+; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v3
+; GFX11TRUE16-NEXT: v_bfe_u32 v19, v11, 16, 1
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v17
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
+; GFX11TRUE16-NEXT: v_dual_cndmask_b32 v10, v10, v18 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v19, v11, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v11
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11TRUE16-NEXT: v_bfe_u32 v11, v17, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v22, 16, 1
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v21, v24, v23
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v9, v0, v1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v12, v18, vcc_lo
-; GFX11TRUE16-NEXT: v_add3_u32 v11, v11, v17, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v21, 16, 1
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v20, v22, v21 :: v_dual_and_b32 v21, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v17, v1, v9 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v18, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v16
+; GFX11TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v8
+; GFX11TRUE16-NEXT: v_bfe_u32 v11, v20, 16, 1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v20
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v18, v21, v19
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v11, v20, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v16, v0, v8
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v17, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v8, v18, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v18
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11TRUE16-NEXT: v_bfe_u32 v9, v16, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v18, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v17, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v17
+; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v16
+; GFX11TRUE16-NEXT: v_add3_u32 v9, v9, v16, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11TRUE16-NEXT: v_add3_u32 v12, v19, v22, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v18, v9, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v21
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v21, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v11, v16, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v22
-; GFX11TRUE16-NEXT: v_add3_u32 v16, v18, v9, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v9
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v8.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v0, v19, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v10.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v12, v1, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v11.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v16, v17, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v19, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v10.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v12.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v11.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v15.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v9
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v16bf16:
@@ -50253,6 +50030,8 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX11TRUE16-NEXT: v_and_b32_e32 v118, 0xffff0000, v0
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v16
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11TRUE16-NEXT: v_and_b32_e32 v103, 0xffff0000, v19
; GFX11TRUE16-NEXT: v_and_b32_e32 v112, 0xffff0000, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v19
@@ -50262,17 +50041,16 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX11TRUE16-NEXT: v_and_b32_e32 v113, 0xffff0000, v18
; GFX11TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(16)
-; GFX11TRUE16-NEXT: v_and_b32_e32 v119, 0xffff0000, v31
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(15)
; GFX11TRUE16-NEXT: v_and_b32_e32 v128, 0xffff0000, v32
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(14)
; GFX11TRUE16-NEXT: v_and_b32_e32 v129, 0xffff0000, v33
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v33
; GFX11TRUE16-NEXT: v_and_b32_e32 v68, 0xffff0000, v13
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(12)
-; GFX11TRUE16-NEXT: v_and_b32_e32 v131, 0xffff0000, v35
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(11)
+; GFX11TRUE16-NEXT: v_and_b32_e32 v132, 0xffff0000, v36
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(10)
; GFX11TRUE16-NEXT: v_and_b32_e32 v133, 0xffff0000, v37
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(9)
@@ -50283,13 +50061,13 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v48
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(5)
; GFX11TRUE16-NEXT: v_and_b32_e32 v146, 0xffff0000, v50
-; GFX11TRUE16-NEXT: v_and_b32_e32 v145, 0xffff0000, v49
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(4)
; GFX11TRUE16-NEXT: v_and_b32_e32 v147, 0xffff0000, v51
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v51
; GFX11TRUE16-NEXT: v_and_b32_e32 v96, 0xffff0000, v7
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11TRUE16-NEXT: v_and_b32_e32 v145, 0xffff0000, v49
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v49
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11TRUE16-NEXT: v_and_b32_e32 v148, 0xffff0000, v55
; GFX11TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v23
@@ -50299,250 +50077,273 @@ define <32 x bfloat> @v_fma_v32bf16(<32 x bfloat> %a, <32 x bfloat> %b, <32 x bf
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v55
; GFX11TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11TRUE16-NEXT: v_and_b32_e32 v84, 0xffff0000, v9
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v48, v7, v23
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v51, v4, v20 :: v_dual_and_b32 v84, 0xffff0000, v9
; GFX11TRUE16-NEXT: v_and_b32_e32 v135, 0xffff0000, v39
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v39
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v49, v6, v22
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v48, v7, v23 :: v_dual_fmac_f32 v49, v6, v22
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v134, v84, v83 :: v_dual_lshlrev_b32 v13, 16, v13
-; GFX11TRUE16-NEXT: v_bfe_u32 v83, v146, 16, 1
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v51, v4, v20 :: v_dual_fmac_f32 v148, v118, v117
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v144, v96, v87 :: v_dual_and_b32 v81, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v148, v118, v117 :: v_dual_and_b32 v81, 0xffff0000, v26
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v144, v96, v87
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v55, v0, v16 :: v_dual_lshlrev_b32 v26, 16, v26
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v145, v98, v97
-; GFX11TRUE16-NEXT: v_or_b32_e32 v84, 0x400000, v146
-; GFX11TRUE16-NEXT: v_add3_u32 v83, v83, v146, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v83, 0x400000, v146
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v18
; GFX11TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v8
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
; GFX11TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v10
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v147, v102, v101 :: v_dual_lshlrev_b32 v10, 16, v10
+; GFX11TRUE16-NEXT: v_bfe_u32 v96, v51, 16, 1
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v50, v5, v21 :: v_dual_and_b32 v71, 0xffff0000, v27
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v38
; GFX11TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v28
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v37, v10, v26 :: v_dual_lshlrev_b32 v28, 16, v28
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v135, v86, v85
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v39, v8, v24
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v133, v82, v81 :: v_dual_and_b32 v70, 0xffff0000, v12
-; GFX11TRUE16-NEXT: v_bfe_u32 v97, v51, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v23, v37, 16, 1
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v135, v86, v85 :: v_dual_lshlrev_b32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; GFX11TRUE16-NEXT: v_or_b32_e32 v97, 0x400000, v51
+; GFX11TRUE16-NEXT: v_add3_u32 v96, v96, v51, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v131, 0xffff0000, v35
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v35, 16, v35
; GFX11TRUE16-NEXT: v_and_b32_e32 v80, 0xffff0000, v11
-; GFX11TRUE16-NEXT: v_and_b32_e32 v132, 0xffff0000, v36
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v36
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v50
-; GFX11TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v133
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v24, 0x400000, v37
-; GFX11TRUE16-NEXT: v_or_b32_e32 v98, 0x400000, v51
-; GFX11TRUE16-NEXT: v_add3_u32 v23, v23, v37, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
-; GFX11TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v27
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v27
-; GFX11TRUE16-NEXT: v_add3_u32 v97, v97, v51, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v17
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v36
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v35, v12, v28
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v132, v80, v71 :: v_dual_and_b32 v67, 0xffff0000, v29
+; GFX11TRUE16-NEXT: v_bfe_u32 v80, v49, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v84, v50, 16, 1
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v36, v11, v27 :: v_dual_lshlrev_b32 v29, 16, v29
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v32, 16, v32
-; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v15
; GFX11TRUE16-NEXT: v_and_b32_e32 v130, 0xffff0000, v34
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v35, v12, v28 :: v_dual_lshlrev_b32 v34, 16, v34
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v36, v11, v27
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v50, v5, v21
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v132, v80, v71 :: v_dual_and_b32 v67, 0xffff0000, v29
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v29
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v34
+; GFX11TRUE16-NEXT: v_or_b32_e32 v81, 0x400000, v49
+; GFX11TRUE16-NEXT: v_or_b32_e32 v85, 0x400000, v50
+; GFX11TRUE16-NEXT: v_add3_u32 v80, v80, v49, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v119, 0xffff0000, v31
+; GFX11TRUE16-NEXT: v_add3_u32 v84, v84, v50, 0x7fff
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v31
+; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v15
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v34, v13, v29 :: v_dual_lshlrev_b32 v15, 16, v15
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v130, v68, v67 :: v_dual_and_b32 v65, 0xffff0000, v30
-; GFX11TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v36
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v34, v13, v29 :: v_dual_fmac_f32 v31, v15, v32
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v119, v64, v128 :: v_dual_and_b32 v66, 0xffff0000, v14
+; GFX11TRUE16-NEXT: v_bfe_u32 v16, v132, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v31, v15, v32 :: v_dual_and_b32 v66, 0xffff0000, v14
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v119, v64, v128
; GFX11TRUE16-NEXT: v_and_b32_e32 v64, 0xffff0000, v52
; GFX11TRUE16-NEXT: v_and_b32_e32 v128, 0xffff0000, v53
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v53
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v129, v66, v65 :: v_dual_lshlrev_b32 v30, 16, v30
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52
; GFX11TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v54
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v54
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v119, 16, 1
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v129, v66, v65 :: v_dual_lshlrev_b32 v30, 16, v30
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v52
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v64, v112, v103
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v38, v9, v25
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v131, v70, v69 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v53, v2, v18
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v119, 16, 1
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v53, v2, v18 :: v_dual_fmac_f32 v54, v1, v17
+; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v119
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v31, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v119, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v119, v119
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v33, v14, v30 :: v_dual_fmac_f32 v52, v3, v19
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v54, v1, v17
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v119
; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v31
; GFX11TRUE16-NEXT: v_bfe_u32 v4, v129, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v119, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v119, v119
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v31, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e64 s0, v31, v31
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v31, v31
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v129
; GFX11TRUE16-NEXT: v_bfe_u32 v6, v33, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v14, v132, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v15, v0, v1, vcc_lo
-; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v149, v2, v3, s0
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v4, v129, 0x7fff
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v129, v129
+; GFX11TRUE16-NEXT: v_add3_u32 v4, v4, v129, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v33
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v129, v129
; GFX11TRUE16-NEXT: v_bfe_u32 v8, v130, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v3, v6, v33, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v150, v14, v132, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v14, v2, v5, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX11TRUE16-NEXT: v_add3_u32 v6, v6, v33, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v130
; GFX11TRUE16-NEXT: v_bfe_u32 v10, v34, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v13, v35, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v8, v130, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v3, v7, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v130, v130
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v33, v33
+; GFX11TRUE16-NEXT: v_add3_u32 v8, v8, v130, 0x7fff
; GFX11TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v34
; GFX11TRUE16-NEXT: v_bfe_u32 v12, v131, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v6, v10, v34, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v10, v13, v35, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v13, v4, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v10, v10, v34, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v130, v130
+; GFX11TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v131
+; GFX11TRUE16-NEXT: v_bfe_u32 v14, v35, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v12, v12, v131, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v35
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v9, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v34, v34
-; GFX11TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v131
-; GFX11TRUE16-NEXT: v_add3_u32 v8, v12, v131, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v35
-; GFX11TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v132
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v34, v6, v11, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v14, v14, v35, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v132
+; GFX11TRUE16-NEXT: v_bfe_u32 v18, v36, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v16, v16, v132, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v11, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v131, v131
-; GFX11TRUE16-NEXT: v_bfe_u32 v19, v36, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v21, v133, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v25, v134, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v134
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v12, v8, v16, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v36
+; GFX11TRUE16-NEXT: v_bfe_u32 v20, v133, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v18, v18, v36, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v133
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v12, v13, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v35, v35
-; GFX11TRUE16-NEXT: v_add3_u32 v19, v19, v36, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v21, v21, v133, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v27, v38, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v25, v25, v134, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v10, v17, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v22, v37, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v20, v20, v133, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v37
+; GFX11TRUE16-NEXT: v_bfe_u32 v24, v134, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v14, v15, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v132, v132
-; GFX11TRUE16-NEXT: v_or_b32_e32 v28, 0x400000, v38
-; GFX11TRUE16-NEXT: v_bfe_u32 v29, v135, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v27, v27, v38, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v30, 0x400000, v135
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v11, v150, v18, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v22, v22, v37, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v134
+; GFX11TRUE16-NEXT: v_bfe_u32 v26, v38, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v24, v24, v134, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v16, v17, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v36, v36
-; GFX11TRUE16-NEXT: v_bfe_u32 v65, v39, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v29, v29, v135, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v66, 0x400000, v39
-; GFX11TRUE16-NEXT: v_bfe_u32 v67, v144, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v19, v20, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v38
+; GFX11TRUE16-NEXT: v_bfe_u32 v28, v135, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v26, v26, v38, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v29, 0x400000, v135
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v18, v19, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v133, v133
-; GFX11TRUE16-NEXT: v_add3_u32 v65, v65, v39, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v68, 0x400000, v144
-; GFX11TRUE16-NEXT: v_bfe_u32 v69, v48, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v67, v67, v144, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v21, v22, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v30, v39, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v28, v28, v135, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v65, 0x400000, v39
+; GFX11TRUE16-NEXT: v_bfe_u32 v66, v144, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v10, v20, v21, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
-; GFX11TRUE16-NEXT: v_or_b32_e32 v70, 0x400000, v48
-; GFX11TRUE16-NEXT: v_bfe_u32 v71, v145, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v69, v69, v48, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v80, 0x400000, v145
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v23, v24, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v30, v30, v39, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v67, 0x400000, v144
+; GFX11TRUE16-NEXT: v_bfe_u32 v68, v48, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v66, v66, v144, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v16, v22, v23, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v134, v134
-; GFX11TRUE16-NEXT: v_bfe_u32 v81, v49, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v71, v71, v145, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v82, 0x400000, v49
-; GFX11TRUE16-NEXT: v_bfe_u32 v85, v50, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v9, v25, v26, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v69, 0x400000, v48
+; GFX11TRUE16-NEXT: v_bfe_u32 v70, v145, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v68, v68, v48, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v71, 0x400000, v145
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v17, v24, v25, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v38, v38
-; GFX11TRUE16-NEXT: v_add3_u32 v81, v81, v49, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v86, 0x400000, v50
-; GFX11TRUE16-NEXT: v_bfe_u32 v87, v147, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v85, v85, v50, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v27, v28, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v70, v70, v145, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v82, v146, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v86, v147, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v87, 0x400000, v147
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v18, v26, v27, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v135, v135
-; GFX11TRUE16-NEXT: v_or_b32_e32 v96, 0x400000, v147
-; GFX11TRUE16-NEXT: v_add3_u32 v87, v87, v147, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v99, v64, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v100, 0x400000, v64
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v8, v29, v30, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v82, v82, v146, 0x7fff
+; GFX11TRUE16-NEXT: v_add3_u32 v86, v86, v147, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v98, v64, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v99, 0x400000, v64
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v19, v28, v29, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v39, v39
-; GFX11TRUE16-NEXT: v_bfe_u32 v101, v52, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v99, v99, v64, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v102, 0x400000, v52
-; GFX11TRUE16-NEXT: v_bfe_u32 v117, v54, 16, 1
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v65, v66, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v100, v52, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v98, v98, v64, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v101, 0x400000, v52
+; GFX11TRUE16-NEXT: v_bfe_u32 v112, v53, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v20, v30, v65, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v144, v144
-; GFX11TRUE16-NEXT: v_add3_u32 v101, v101, v52, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v118, 0x400000, v54
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v55, 16, 1
-; GFX11TRUE16-NEXT: v_add3_u32 v117, v117, v54, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v67, v68, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v100, v100, v52, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v1
+; GFX11TRUE16-NEXT: v_add3_u32 v112, v112, v53, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v1, v148, 16, 1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v66, v67, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v55
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v55, 0x7fff
-; GFX11TRUE16-NEXT: v_bfe_u32 v119, v148, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v31, 0x400000, v148
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v21, v69, v70, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v0
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v8
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v20
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v68, v69, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v145, v145
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.l, v20.h
-; GFX11TRUE16-NEXT: v_add3_u32 v119, v119, v148, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.l, v19.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.l, v21.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v6, v71, v80, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v54, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v148, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v6
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v70, v71, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v49, v49
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.l, v18.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.l, v17.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.l, v16.h
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.l, v34.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v22, v81, v82, vcc_lo
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v128, v114, v113
+; GFX11TRUE16-NEXT: v_or_b32_e32 v113, 0x400000, v53
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v54, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v9
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v80, v81, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v146, v146
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.l, v33.h
-; GFX11TRUE16-NEXT: v_mov_b16_e64 v15.l, v149.h
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.l, v22.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v5, v83, v84, vcc_lo
+; GFX11TRUE16-NEXT: v_bfe_u32 v102, v128, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v103, 0x400000, v128
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v18
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v24
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v82, v83, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v50, v50
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v23, v85, v86, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v102, v102, v128, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v24, v55, 16, 1
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v21
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v25
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v84, v85, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v147, v147
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.l, v23.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v4, v87, v96, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v25, 0x400000, v148
+; GFX11TRUE16-NEXT: v_add3_u32 v24, v24, v55, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v4
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v26
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v86, v87, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v51, v51
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v128, v114, v113
-; GFX11TRUE16-NEXT: v_bfe_u32 v113, v53, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v114, 0x400000, v53
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v97, v98, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v54
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v7
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v10
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v27
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v96, v97, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v64, v64
-; GFX11TRUE16-NEXT: v_bfe_u32 v103, v128, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v112, 0x400000, v128
-; GFX11TRUE16-NEXT: v_add3_u32 v113, v113, v53, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.l, v24.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v99, v100, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v27, 0x400000, v55
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v17
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v19
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v29, v98, v99, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v52, v52
-; GFX11TRUE16-NEXT: v_add3_u32 v103, v103, v128, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v25, v101, v102, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v128, v128
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v32, v116, v115
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.l, v25.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v103, v112, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v22
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v23
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v29
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v30, v100, v101, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v128, v128
+; GFX11TRUE16-NEXT: v_bfe_u32 v114, v32, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v115, 0x400000, v32
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v28
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v31, v102, v103, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v53, v53
-; GFX11TRUE16-NEXT: v_bfe_u32 v115, v32, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v116, 0x400000, v32
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v26, v113, v114, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v115, v115, v32, 0x7fff
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.l, v26.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v27, v117, v118, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v28, v0, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_add3_u32 v114, v114, v32, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v30
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v3.h, v22.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v31
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v33, v112, v113, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v115, v116, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v4.h, v21.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v5.h, v20.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v6.h, v19.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v7.h, v18.l
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v114, v115, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v148, v148
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v27.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v119, v31, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v28.h
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v8.h, v17.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v9.h, v16.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v10.h, v39.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v11.h, v38.l
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v25, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v54, v54
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v12.h, v37.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v13.h, v36.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v14.h, v35.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v25, 16, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v26, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v55, v55
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v2
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v33
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v2.h, v23.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v24, v24, v27, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v26.l
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v15.h, v34.l
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v24
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v25.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fma_v32bf16:
@@ -51023,41 +50824,23 @@ define bfloat @v_fmuladd_bf16(bfloat %a, bfloat %b, bfloat %c) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11TRUE16-LABEL: v_fmuladd_bf16:
-; GFX11TRUE16: ; %bb.0:
-; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v2, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11FAKE16-LABEL: v_fmuladd_bf16:
-; GFX11FAKE16: ; %bb.0:
-; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX11FAKE16-NEXT: v_bfe_u32 v0, v2, 16, 1
-; GFX11FAKE16-NEXT: v_or_b32_e32 v1, 0x400000, v2
-; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
-; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_fmuladd_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX11-NEXT: v_bfe_u32 v0, v2, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v1, 0x400000, v2
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v0, v0, v2, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_fmuladd_bf16:
; GFX1250: ; %bb.0:
@@ -51202,12 +50985,12 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
; GFX11TRUE16-LABEL: v_fmuladd_v2bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11TRUE16-NEXT: v_dual_fmac_f32 v2, v0, v1 :: v_dual_fmac_f32 v3, v5, v4
; GFX11TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
@@ -51218,12 +51001,14 @@ define <2 x bfloat> @v_fmuladd_v2bf16(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfl
; GFX11TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v4, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v5, vcc_lo
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmuladd_v2bf16:
@@ -51447,40 +51232,43 @@ define <3 x bfloat> @v_fmuladd_v3bf16(<3 x bfloat> %a, <3 x bfloat> %b, <3 x bfl
; GFX11TRUE16-LABEL: v_fmuladd_v3bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v5, v1, v3 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v5, 16, 1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v6, v8, v7
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v0, v2
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_bfe_u32 v7, v6, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v0, 0x400000, v6
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v7, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v4, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
-; GFX11TRUE16-NEXT: v_bfe_u32 v1, v6, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v0, v7, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc_lo
+; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v4
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmuladd_v3bf16:
@@ -51764,48 +51552,52 @@ define <4 x bfloat> @v_fmuladd_v4bf16(<4 x bfloat> %a, <4 x bfloat> %b, <4 x bfl
; GFX11TRUE16-LABEL: v_fmuladd_v4bf16:
; GFX11TRUE16: ; %bb.0:
; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v1
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v0
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v5
; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v5, v1, v3
-; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_lshlrev_b32 v7, 16, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v4
-; GFX11TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v2
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_dual_fmac_f32 v6, v8, v7 :: v_dual_and_b32 v7, 0xffff0000, v4
+; GFX11TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11TRUE16-NEXT: v_add3_u32 v4, v9, v6, 0x7fff
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX11TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
-; GFX11TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
-; GFX11TRUE16-NEXT: v_fmac_f32_e32 v3, v0, v1
-; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v6
; GFX11TRUE16-NEXT: v_fmac_f32_e32 v7, v10, v8
-; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11TRUE16-NEXT: v_fmac_f32_e32 v4, v0, v2
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v9, v6, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
+; GFX11TRUE16-NEXT: v_bfe_u32 v2, v7, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v7
; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v6
-; GFX11TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11TRUE16-NEXT: v_bfe_u32 v0, v7, 16, 1
-; GFX11TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11TRUE16-NEXT: v_add3_u32 v5, v9, v3, 0x7fff
-; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v7, 0x7fff
-; GFX11TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11TRUE16-NEXT: v_add3_u32 v2, v2, v7, 0x7fff
+; GFX11TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc_lo
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v7, v0, v10, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc_lo
-; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX11TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11FAKE16-LABEL: v_fmuladd_v4bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/bitop3.ll b/llvm/test/CodeGen/AMDGPU/bitop3.ll
index a5da16543daf6..2caf9c262977d 100644
--- a/llvm/test/CodeGen/AMDGPU/bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitop3.ll
@@ -786,14 +786,25 @@ define amdgpu_ps half @test_xor3_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0x96
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: test_xor3_b16:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-GISEL-NEXT: v_nop
-; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: v_xor3_b32 v0, v0, v1, v2
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: test_xor3_b16:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-FAKE16-NEXT: v_xor3_b32 v0, v0, v1, v2
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: test_xor3_b16:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
%xor1 = xor i16 %a, %b
%xor2 = xor i16 %xor1, %c
%ret_cast = bitcast i16 %xor2 to half
@@ -829,14 +840,25 @@ define amdgpu_ps half @test_or3_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xfe
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: test_or3_b16:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-GISEL-NEXT: v_nop
-; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: test_or3_b16:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-FAKE16-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: test_or3_b16:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
%or1 = or i16 %a, %b
%or2 = or i16 %or1, %c
%ret_cast = bitcast i16 %or2 to half
@@ -872,14 +894,25 @@ define amdgpu_ps half @test_and_or_b16(i16 %a, i16 %b, i16 %c) {
; GFX1250-SDAG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, v1.l bitop3:0xec
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: test_and_or_b16:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-GISEL-NEXT: v_nop
-; GFX1250-GISEL-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: test_and_or_b16:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-FAKE16-NEXT: v_nop
+; GFX1250-GISEL-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, v0, v1, v2
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+;
+; GFX1250-GISEL-TRUE16-LABEL: test_and_or_b16:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-GISEL-TRUE16-NEXT: v_nop
+; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-GISEL-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
%and1 = and i16 %a, %b
%or1 = or i16 %and1, %c
%ret_cast = bitcast i16 %or1 to half
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index ef17b1dd7527b..ae7dde0fec38b 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -4966,63 +4966,120 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB16_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5067,57 +5124,108 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB16_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -5372,62 +5480,118 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB17_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5471,56 +5635,106 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB17_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index fe0cc95dc0a51..3b2853fdc5e1e 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -4071,63 +4071,120 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4172,57 +4229,108 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4479,62 +4587,118 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4578,56 +4742,106 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_max_f32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index 6ba9f61ba7faf..e280847800916 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -4071,63 +4071,120 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4172,57 +4229,108 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_min_f32_e32 v0, v0, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4479,62 +4587,118 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_b32 s4, s16, -4
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_and_b32 s4, s16, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-NEXT: s_not_b32 s6, s5
-; GFX12-NEXT: s_mov_b32 s5, 0
-; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4578,56 +4742,106 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_addk_i32 s16, 0x200
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-NEXT: s_and_b32 s4, s16, -4
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-NEXT: s_and_b32 s4, s16, 3
-; GFX11-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-NEXT: s_not_b32 s6, s5
-; GFX11-NEXT: s_mov_b32 s5, 0
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_min_f32_e32 v0, v0, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
+; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
+; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index ad9f317c8842b..65218bec1b145 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -6476,25 +6476,36 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
; GFX11-TRUE16-NEXT: s_addc_u32 s3, s3, external_void_func_v16i8 at rel32@hi+12
; GFX11-TRUE16-NEXT: s_mov_b32 s32, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_load_b128 v[16:19], off, s[4:7], 0
+; GFX11-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
; GFX11-TRUE16-NEXT: s_mov_b64 s[6:7], s[0:1]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[2:3]
; GFX11-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
index af40349edb097..d31e9ba894f6f 100644
--- a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
+++ b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
@@ -2966,25 +2966,25 @@ define amdgpu_cs void @amdgpu_cs_v8i1(<8 x i1> %arg0) {
;
; GFX11-TRUE16-LABEL: amdgpu_cs_v8i1:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v6.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v5.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v4.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v6.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v5.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v4.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 3, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 3, v7.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.h, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v2.h, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
@@ -3030,23 +3030,23 @@ define amdgpu_cs void @amdgpu_cs_v8i1(<8 x i1> %arg0) {
; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v3.l, v6.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 3, v7.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 1, v5.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v0.h, v6.l, 1
+; GFX1250-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v7.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v5.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 2, v3.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v4.l, v2.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v1.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v3.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.l, v3.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v2.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v0.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v2.l, v0.h, 3 bitop3:0xec
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.h, 3 bitop3:0xec
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 15 bitop3:0xec
@@ -3189,53 +3189,53 @@ define amdgpu_cs void @amdgpu_cs_v16i1(<16 x i1> %arg0) {
;
; GFX11-TRUE16-LABEL: amdgpu_cs_v16i1:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b16 v10.l, v10.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 1, v9.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, v8.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v6.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v5.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v4.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v10.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v13.l
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v12.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 1, v9.l
+; GFX11-TRUE16-NEXT: v_and_b16 v6.h, v8.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 3, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 1, v5.l
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v6.h, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v6.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v14.l, v14.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 1, v13.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.l, v12.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 3, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 2, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v8.l, v9.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 3, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v14.l, 1
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.h, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 3, v7.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 3, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 2, v14.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v12.l, v13.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v5.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v7.l, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 3, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v3.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.h, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v12.h, v10.h
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v8.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v7.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v2.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v4.l, v3.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 15
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 15
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 4, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 12, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
@@ -3303,40 +3303,40 @@ define amdgpu_cs void @amdgpu_cs_v16i1(<16 x i1> %arg0) {
; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v6.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 3, v7.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v4.h, v10.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v8.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 2, v2.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v0.h, v6.l, 1
+; GFX1250-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v7.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v3.l, v10.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v4.h, v14.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 1, v9.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 3, v11.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v4.h, 2, v4.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 2, v3.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 1, v5.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v5.l, v8.l, 1
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.l, 3, v15.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v4.h, 2, v4.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.h, 1, v13.l
-; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v2.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 1, v5.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v5.l, v14.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v9.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.l, 2, v5.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.l, v1.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v3.h, 3, v2.h bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_or_b16 v3.h, v5.h, v4.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v4.l, v2.l, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v5.l, 3, v3.h bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_or_b16 v3.l, v5.h, v3.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v4.l, v6.l, v4.h
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.h, v12.l, v6.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v4.l, v6.l, v5.l
-; GFX1250-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v0.h, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v2.h, 15, v3.h bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.h, v4.l, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v2.l, v0.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v3.h, 15, v3.l bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v4.h, v4.l, 3 bitop3:0xec
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.h, 3 bitop3:0xec
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 12, v1.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 12, v2.l
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 15 bitop3:0xec
; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
@@ -3592,94 +3592,94 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
;
; GFX11-TRUE16-LABEL: amdgpu_cs_v32i1:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b16 v26.l, v26.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.l, 1, v25.l
-; GFX11-TRUE16-NEXT: v_and_b16 v24.l, v24.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v20.h, v22.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v21.l, 1, v21.l
-; GFX11-TRUE16-NEXT: v_and_b16 v20.l, v20.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v17.h, v18.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 1, v17.l
-; GFX11-TRUE16-NEXT: v_and_b16 v16.l, v16.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v10.l, v10.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v30.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 1, v29.l
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v28.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 3, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v24.l, 1
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v2.h, v26.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 1, v21.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.h, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 3, v27.l
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v5.h, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 2, v2.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v22.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v7.h, v20.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 1, v17.l
+; GFX11-TRUE16-NEXT: v_and_b16 v9.h, v16.l, 1
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT: v_and_b16 v3.h, v4.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 3, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v6.h
+; GFX11-TRUE16-NEXT: v_and_b16 v7.h, v18.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.h, v8.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v6.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 3, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 2, v7.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v5.h, v4.h
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, v8.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v6.h, v7.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 1, v13.l
+; GFX11-TRUE16-NEXT: v_and_b16 v7.h, v12.l, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 1, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, v8.l, 1
+; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v4.h, v3.h
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, v10.l, 1
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, v6.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v5.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v4.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v2.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 1, v5.l
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 1
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 1
-; GFX11-TRUE16-NEXT: v_and_b16 v30.l, v30.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.l, 1, v29.l
-; GFX11-TRUE16-NEXT: v_and_b16 v28.l, v28.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v25.h, 3, v27.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v26.l, 2, v26.l
-; GFX11-TRUE16-NEXT: v_or_b16 v22.h, v24.l, v25.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v22.l, 3, v23.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v20.h, 2, v20.h
-; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v20.l, v21.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v18.h, 3, v19.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 2, v17.h
-; GFX11-TRUE16-NEXT: v_or_b16 v16.l, v16.l, v17.l
-; GFX11-TRUE16-NEXT: v_and_b16 v14.l, v14.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.l, 1, v13.l
-; GFX11-TRUE16-NEXT: v_and_b16 v12.l, v12.l, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 3, v11.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 2, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v5.h, v8.l, v9.l
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v14.l, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 3, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 2, v4.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v6.h
+; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v9.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 3, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 2, v6.l
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v28.h, 3, v31.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v29.h, 2, v30.l
-; GFX11-TRUE16-NEXT: v_or_b16 v28.l, v28.l, v29.l
-; GFX11-TRUE16-NEXT: v_or_b16 v25.h, v25.h, v26.l
-; GFX11-TRUE16-NEXT: v_and_b16 v21.h, v22.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v18.l, v22.l, v20.h
-; GFX11-TRUE16-NEXT: v_and_b16 v16.h, v16.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v17.h, v18.h, v17.h
-; GFX11-TRUE16-NEXT: v_and_b16 v16.l, v16.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 3, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 2, v14.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v12.l, v13.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v9.h, v10.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v5.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v7.l, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, v0.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 3, v15.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v8.h, v4.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v7.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v7.l, v6.l
+; GFX11-TRUE16-NEXT: v_and_b16 v4.l, v4.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v26.h, v28.h, v29.h
-; GFX11-TRUE16-NEXT: v_and_b16 v24.h, v28.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v19.l, v21.h, v25.h
-; GFX11-TRUE16-NEXT: v_or_b16 v16.h, v16.h, v18.l
-; GFX11-TRUE16-NEXT: v_or_b16 v15.h, v16.l, v17.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v12.h, v10.h
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, v8.h, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v24.h, v24.h, v26.h
-; GFX11-TRUE16-NEXT: v_and_b16 v14.h, v19.l, 15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 4, v16.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v15.h, 15
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v9.h, v5.h
+; GFX11-TRUE16-NEXT: v_and_b16 v5.h, v6.h, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, v1.h, 15
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 4, v2.h
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, v3.h, 15
+; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v5.h, v3.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, v1.l, 15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 4, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, v0.l, 15
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 12, v24.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v14.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v1.h, v16.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 12, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 12, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 12, v3.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v2.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
@@ -3793,79 +3793,79 @@ define amdgpu_cs void @amdgpu_cs_v32i1(<32 x i1> %arg0) {
; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-TRUE16-NEXT: v_nop
; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v18.h, 3, v19.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v19.l, v22.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v18.l, v18.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.h, 3, v23.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.h, 1, v21.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.l, 1, v17.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.l, 2, v19.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v6.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v18.l, 2, v18.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 3, v7.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v16.l, v16.l, v17.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v16.h, v17.h, v19.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v17.h, v20.l, v19.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_and_b16 v17.l, v26.l, 1
+; GFX1250-TRUE16-NEXT: v_and_b16 v1.h, v22.l, 1
+; GFX1250-TRUE16-NEXT: v_and_b16 v0.h, v18.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 3, v23.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 3, v19.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v4.h, 1, v21.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 2, v1.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 2, v0.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 1, v25.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v6.h, v24.l, 1
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.h, v20.l, v4.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v26.l, 1
+; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v17.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v7.h, 3, v27.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v8.h, v30.l, 1
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 2, v3.h
-; GFX1250-TRUE16-NEXT: v_or_b16 v18.l, v18.h, v18.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v18.h, v24.l, 1
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v16.h, v17.h, v16.h, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.h, 1, v25.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.l, 3, v27.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.l, 2, v17.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v19.h, v30.l, 1
-; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v3.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 1, v5.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v5.l, v10.l, 1
-; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v14.l, 1
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v17.h, v18.h, 3, v17.h bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_or_b16 v17.l, v19.l, v17.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v18.h, 3, v31.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.l, 2, v19.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v19.h, 1, v29.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.h, v1.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v16.l, v2.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.h, v6.h, 3, v5.h bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 3, v31.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v3.h, v7.h, v3.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.h, 2, v8.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v2.h, v0.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 1, v29.l
; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 1, v9.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v4.h, v8.l, 1
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.l, 3, v11.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.l, 2, v5.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.h, 3, v15.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v4.h, 15, v3.h bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_or_b16 v4.h, v5.h, v6.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v6.l, 1
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v28.l, v2.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.l, 3, v7.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v6.h, v14.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.l, 1, v5.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v7.l, 1, v13.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v17.l, v17.h, 15, v17.l bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v16.l, v16.l, v18.l, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v17.h, v18.h, v19.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v18.l, v28.l, v19.h, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v2.h, v4.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_and_b16 v7.l, v8.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v7.h, 3, v11.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v8.l, 3, v15.l
+; GFX1250-TRUE16-NEXT: v_or_b16 v4.h, v6.l, v5.h
+; GFX1250-TRUE16-NEXT: v_and_b16 v5.h, v10.l, 1
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.l, 1, v9.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v6.h, 2, v6.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v8.h, 1, v13.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 3, v3.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v5.h, 2, v5.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 2, v2.l
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 1, v1.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v4.l, v1.h, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v4.h, 3, v3.h bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_or_b16 v4.l, v6.l, v5.l
-; GFX1250-TRUE16-NEXT: v_or_b16 v4.h, v6.h, v5.h
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v5.l, v12.l, v7.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v18.l, v17.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.l, v4.l, v5.l, 1 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v5.l, v7.l, 3, v6.l bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_or_b16 v5.h, v7.h, v5.h
+; GFX1250-TRUE16-NEXT: v_or_b16 v6.l, v8.l, v6.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v6.h, v12.l, v8.h, 1 bitop3:0xec
; GFX1250-TRUE16-NEXT: v_or_b16 v2.l, v3.l, v2.l
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 1 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v0.h, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v3.h, 15, v4.l bitop3:0xc8
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v5.l, v4.h, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v16.h, 4, v16.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v17.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v4.l, v4.h, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v5.l, 15, v5.h bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v4.l, v6.h, v6.l, 3 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 4, v1.h
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.h, 12, v2.h
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v2.l, 3 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v0.h, 4, v0.h
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.h, 12, v1.h
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v16.l, v16.h, 15 bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_or_b16 v2.h, v2.h, v17.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 15 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v1.l, 4, v1.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, 12, v4.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.h, v1.h, 15 bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v3.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 15 bitop3:0xec
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v1.l
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.h, v2.l, v2.h, 0xff bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v2.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.h, v1.h, 0xff bitop3:0xec
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v0.h, 0xff bitop3:0xec
-; GFX1250-TRUE16-NEXT: global_store_b32 v[0:1], v1, off
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0xff bitop3:0xec
+; GFX1250-TRUE16-NEXT: global_store_b32 v[0:1], v0, off
; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: amdgpu_cs_v32i1:
diff --git a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
index 2ce5ae6a93bb9..9d0cfd1f10c7f 100644
--- a/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine_andor_with_cmps.ll
@@ -2568,9 +2568,9 @@ define i1 @test131(i16 %arg1, i32 %arg2) {
; GFX11-TRUE16-LABEL: test131:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_gt_u32_e32 vcc_lo, 10, v1
-; GFX11-TRUE16-NEXT: v_cmp_gt_u16_e64 s0, 10, v0.l
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_u16_e32 vcc_lo, 10, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_u32_e64 s0, 10, v1
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index abeb42213466b..7634d16c92788 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -299,23 +299,14 @@ define <2 x float> @v_uitofp_v2i8_to_v2f32(i16 %arg0) nounwind {
; GFX9-NEXT: v_mov_b32_e32 v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_uitofp_v2i8_to_v2f32:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte0_e32 v0, v1
-; GFX11-TRUE16-NEXT: v_cvt_f32_ubyte1_e32 v1, v1
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_uitofp_v2i8_to_v2f32:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
-; GFX11-FAKE16-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_uitofp_v2i8_to_v2f32:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
+; GFX11-NEXT: v_cvt_f32_ubyte1_e32 v1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val = bitcast i16 %arg0 to <2 x i8>
%cvt = uitofp <2 x i8> %val to <2 x float>
ret <2 x float> %cvt
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index 87ed134209657..e201f1c02e60c 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -2203,11 +2203,11 @@ define half @fmul_select_f16_test6(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test6:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4200
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc800, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4200
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0xc800, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_f16_test6:
@@ -2313,11 +2313,11 @@ define half @fmul_select_f16_test7(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-SDAG-TRUE16-LABEL: fmul_select_f16_test7:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4800, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc400
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0x4800, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_f16_test7:
@@ -2401,9 +2401,9 @@ define half @fmul_select_f16_test8(half %x, i32 %bool.arg1, i32 %bool.arg2) {
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x8000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, 0x8000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_f16_test8:
@@ -2837,22 +2837,23 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test1:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3f80
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_bf16_test1:
@@ -3023,22 +3024,23 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test2:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3f80
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3f80
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3f00, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x3f00, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_bf16_test2:
@@ -3274,16 +3276,16 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x4000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x4000, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
@@ -3298,8 +3300,11 @@ define <2 x bfloat> @fmul_select_v2bf16_test3(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_v2bf16_test3:
@@ -3573,16 +3578,16 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, 0x3f80
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v2, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, v1, v3
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, 0x3f00, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, 0x3f00, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v2 :: v_dual_mul_f32 v1, v3, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
@@ -3597,8 +3602,11 @@ define <2 x bfloat> @fmul_select_v2bf16_test4(<2 x bfloat> %x, <2 x i32> %bool.a
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_v2bf16_test4:
@@ -3808,22 +3816,23 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test5:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4100
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4100
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_bf16_test5:
@@ -3996,22 +4005,23 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test6:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4040
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4040
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc100, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xc100, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_bf16_test6:
@@ -4183,22 +4193,23 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test7:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc080
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc080
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x4100, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4100, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_bf16_test7:
@@ -4380,7 +4391,7 @@ define bfloat @fmul_select_bf16_test8(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_bf16_test8:
@@ -4550,22 +4561,23 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test9:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xc200
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc200
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xc180, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xc180, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_bf16_test9:
@@ -4738,22 +4750,23 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xdb80
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xdb80
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0xe000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xe000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_bf16_test10_sel_log2val_pos65_pos56:
@@ -4926,22 +4939,23 @@ define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %b
; GFX11-SDAG-TRUE16-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4c00
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4c00
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, 0x3480, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x3480, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: fmul_select_bf16_test11_sel_log2val_neg22_pos25:
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
index 015185fcea7d9..a5850d54447c0 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
@@ -258,10 +258,7 @@ define float @divergent_vec_f16_neg_0(half %a) {
; GFX11-TRUE16-LABEL: divergent_vec_f16_neg_0:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0x8000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x8000
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: divergent_vec_f16_neg_0:
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 028ec031941d3..275c09bfa493f 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -3205,7 +3205,7 @@ define void @test_dynamic_stackalloc_device_divergent_non_standard_size_i16(i16
; GFX11-GISEL-NEXT: scratch_store_b32 off, v1, s33
; GFX11-GISEL-NEXT: scratch_store_b32 off, v2, s33 offset:4
; GFX11-GISEL-NEXT: s_mov_b32 exec_lo, s0
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-GISEL-NEXT: s_add_i32 s32, s32, 16
; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: s_mov_b32 s0, s32
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
index 99a0241f2d4a6..3819863d402ce 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
@@ -1247,10 +1247,9 @@ define <2 x bfloat> @v_test_canonicalize_reg_k_v2bf16(bfloat %val) #1 {
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v1.h, 0x4000
-; REAL16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; REAL16-NEXT: v_mov_b16_e32 v0.h, 0x4000
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_pk_mul_bf16 v0, 1.0, v1 op_sel_hi:[0,1]
+; REAL16-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; REAL16-NEXT: s_set_pc_i64 s[30:31]
%vec0 = insertelement <2 x bfloat> poison, bfloat %val, i32 0
%vec1 = insertelement <2 x bfloat> %vec0, bfloat 2.0, i32 1
@@ -1273,10 +1272,10 @@ define <2 x bfloat> @v_test_canonicalize_k_reg_v2bf16(bfloat %val) #1 {
; REAL16: ; %bb.0:
; REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: v_mov_b16_e32 v1.l, 0x4000
-; REAL16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; REAL16-NEXT: v_mov_b16_e32 v0.h, v0.l
+; REAL16-NEXT: v_mov_b16_e32 v0.l, 0x4000
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; REAL16-NEXT: v_pk_mul_bf16 v0, 1.0, v1 op_sel_hi:[0,1]
+; REAL16-NEXT: v_pk_mul_bf16 v0, 1.0, v0 op_sel_hi:[0,1]
; REAL16-NEXT: s_set_pc_i64 s[30:31]
%vec0 = insertelement <2 x bfloat> poison, bfloat 2.0, i32 0
%vec1 = insertelement <2 x bfloat> %vec0, bfloat %val, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
index d55886e13e998..1d237068dab04 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
@@ -3040,11 +3040,11 @@ define <4 x half> @v_test_canonicalize_reg_undef_reg_reg_v4f16(half %val0, half
; GFX11-TRUE16-LABEL: v_test_canonicalize_reg_undef_reg_reg_v4f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: v_pk_max_f16 v1, v1, v1
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_test_canonicalize_reg_undef_reg_reg_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index 300d9086ad50f..e16142458f549 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -41,9 +41,9 @@ define amdgpu_ps i16 @s_copysign_f16(half inreg %mag, half inreg %sign) {
;
; GFX11-TRUE16-LABEL: s_copysign_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, s1
+; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -200,17 +200,10 @@ define amdgpu_ps i16 @s_test_copysign_f16_0_mag(half inreg %sign) {
; GFX9-NEXT: s_and_b32 s0, s0, 0x8000
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_test_copysign_f16_0_mag:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b32_e64 v0, 0x8000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_test_copysign_f16_0_mag:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_test_copysign_f16_0_mag:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half 0.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -236,20 +229,12 @@ define amdgpu_ps i16 @s_test_copysign_f16_1_mag(half inreg %sign) {
; GFX9-NEXT: s_or_b32 s0, s0, 0x3c00
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_test_copysign_f16_1_mag:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b32_e64 v0, 0x8000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x3c00, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_test_copysign_f16_1_mag:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 0x3c00
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_test_copysign_f16_1_mag:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, 0x3c00
+; GFX11-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half 1.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -283,9 +268,10 @@ define i16 @s_test_copysign_f16_10_mag(half inreg %sign) {
; GFX11-TRUE16-LABEL: s_test_copysign_f16_10_mag:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e64 v0, 0x8000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x4900, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, 0x4900
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: s_test_copysign_f16_10_mag:
@@ -320,20 +306,12 @@ define amdgpu_ps i16 @s_test_copysign_f16_neg1_mag(half inreg %sign) {
; GFX9-NEXT: s_or_b32 s0, s0, 0x3c00
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_test_copysign_f16_neg1_mag:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b32_e64 v0, 0x8000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x3c00, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_test_copysign_f16_neg1_mag:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 0x3c00
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_test_copysign_f16_neg1_mag:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, 0x3c00
+; GFX11-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half -1.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -358,20 +336,12 @@ define amdgpu_ps i16 @s_test_copysign_f16_neg10_mag(half inreg %sign) {
; GFX9-NEXT: s_or_b32 s0, s0, 0x4900
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_test_copysign_f16_neg10_mag:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b32_e64 v0, 0x8000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x4900, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_test_copysign_f16_neg10_mag:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 0x4900
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_test_copysign_f16_neg10_mag:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, 0x4900
+; GFX11-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half -10.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -1021,101 +991,52 @@ define half @v_copysign_out_f16_mag_f64_sign_f16(double %mag, half %sign) {
; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_copysign_out_f16_mag_f64_sign_f16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0x1ff, v1, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v1, 20, 11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-TRUE16-NEXT: v_sub_nc_u32_e32 v4, 0x3f1, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0xfffffc10, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0xffe, v3, v0
-; GFX11-TRUE16-NEXT: v_med3_i32 v3, v4, 0, 13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x1000, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v3, v5
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v4
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v1, 12, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 7, v3
-; GFX11-TRUE16-NEXT: v_cmp_lt_i32_e32 vcc_lo, 5, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, 0x7e00
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX11-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, 0x7c00, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v1
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_copysign_out_f16_mag_f64_sign_f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0x1ff, v1, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v1, 20, 11
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT: v_sub_nc_u32_e32 v4, 0x3f1, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0xfffffc10, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, 0xffe, v3, v0
-; GFX11-FAKE16-NEXT: v_med3_i32 v3, v4, 0, 13
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x1000, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v3, v5
-; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v4, v1, 12, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 7, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX11-FAKE16-NEXT: v_cmp_lt_i32_e32 vcc_lo, 5, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0x7e00 :: v_dual_add_nc_u32 v3, v3, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, 0x7c00, v3, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v4, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_copysign_out_f16_mag_f64_sign_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_or_b32 v0, 0x1ff, v1, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-NEXT: v_bfe_u32 v1, v1, 20, 11
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: v_sub_nc_u32_e32 v4, 0x3f1, v1
+; GFX11-NEXT: v_add_nc_u32_e32 v1, 0xfffffc10, v1
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_and_or_b32 v0, 0xffe, v3, v0
+; GFX11-NEXT: v_med3_i32 v3, v4, 0, 13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v4, 0x1000, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v3, v5
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_lshl_or_b32 v4, v1, 12, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_b32_e32 v4, 7, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 2, v3
+; GFX11-NEXT: v_cmp_lt_i32_e32 vcc_lo, 5, v4
+; GFX11-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v4
+; GFX11-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX11-NEXT: v_cmp_gt_i32_e32 vcc_lo, 31, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX11-NEXT: v_dual_mov_b32 v4, 0x7e00 :: v_dual_add_nc_u32 v3, v3, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, 0x7c00, v3, vcc_lo
+; GFX11-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7c00, v4, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0x40f, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, v0, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%mag.trunc = fptrunc double %mag to half
%result = call half @llvm.copysign.f16(half %mag.trunc, half %sign)
ret half %result
@@ -1300,8 +1221,8 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f64_sign_f16(double inreg %mag, hal
; GFX11-TRUE16-NEXT: s_cmpk_eq_i32 s3, 0x40f
; GFX11-TRUE16-NEXT: s_cselect_b32 s0, s0, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -2610,13 +2531,22 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f16_sign_f32(half inreg %mag, float
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_copysign_out_f16_mag_f16_sign_f32:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshrrev_b32_e64 v0, 16, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_copysign_out_f16_mag_f16_sign_f32:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_copysign_out_f16_mag_f16_sign_f32:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e64 v0, 16, s1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%sign.trunc = fptrunc float %sign to half
%out = call half @llvm.copysign.f16(half %mag, half %sign.trunc)
%cast = bitcast half %out to i16
@@ -2650,13 +2580,22 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f16_sign_f64(half inreg %mag, doubl
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_copysign_out_f16_mag_f16_sign_f64:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_lshrrev_b32_e64 v0, 16, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_copysign_out_f16_mag_f16_sign_f64:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e64 v1, 16, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, v1
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_copysign_out_f16_mag_f16_sign_f64:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e64 v0, 16, s2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%sign.trunc = fptrunc double %sign to half
%out = call half @llvm.copysign.f16(half %mag, half %sign.trunc)
%cast = bitcast half %out to i16
@@ -2693,9 +2632,10 @@ define amdgpu_ps i16 @s_copysign_out_f16_mag_f32_sign_f16(float inreg %mag, half
;
; GFX11-TRUE16-LABEL: s_copysign_out_f16_mag_f32_sign_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s1
+; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v1.l, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, s1
+; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -6627,17 +6567,10 @@ define amdgpu_ps i32 @s_copysign_f16_0_f16(half inreg %sign) {
; GFX9-NEXT: s_and_b32 s0, s0, 0x8000
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: s_copysign_f16_0_f16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_and_b32_e64 v0, 0x8000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: s_copysign_f16_0_f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: s_copysign_f16_0_f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-NEXT: ; return to shader part epilog
%op = call half @llvm.copysign.f16(half 0.0, half %sign)
%cast = bitcast half %op to i16
%zext = zext i16 %cast to i32
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
index 25d9a85f6346b..40dfca926d845 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.bf16.ll
@@ -184,11 +184,7 @@ define <2 x bfloat> @v_rsq_bf16_multi_use(bfloat %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(TRANS32_DEP_1)
-; GFX1250-TRUE16-NEXT: v_rsq_bf16_e32 v1.h, v1.l
-; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-TRUE16-NEXT: v_rsq_bf16_e32 v0.h, v0.l
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: v_rsq_bf16_multi_use:
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
index 6f2bf9a4baa07..043f526c8f125 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.f16.ll
@@ -1931,9 +1931,9 @@ define half @v_fdiv_f16_arcp(half %x, half %y) {
; GFX11-TRUE16-LABEL: v_fdiv_f16_arcp:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_fdiv_f16_arcp:
@@ -1982,9 +1982,9 @@ define half @v_fdiv_f16_afn_nsz(half %x, half %y) {
; GFX11-TRUE16-LABEL: v_fdiv_f16_afn_nsz:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_fdiv_f16_afn_nsz:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 1c851a66486d4..35c407758254b 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -11940,58 +11940,114 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB46_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12036,54 +12092,106 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB46_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12283,73 +12391,131 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB47_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: flat_load_dword v3, v[0:1]
-; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX942-NEXT: s_mov_b32 s0, 0xffff
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: flat_load_dword v3, v[0:1]
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: s_mov_b32 s0, 0xffff
; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
@@ -12384,56 +12550,110 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB47_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12639,61 +12859,119 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB48_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12741,56 +13019,110 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB48_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -13696,32 +14028,33 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13729,7 +14062,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB51_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -13814,40 +14147,41 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB51_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -14336,56 +14670,111 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB53_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14429,52 +14818,103 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB53_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14669,62 +15109,121 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14771,56 +15270,110 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index 84703f1b3c950..d4f94afe193a6 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -2802,27 +2802,29 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB18_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -2831,7 +2833,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB18_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
@@ -2840,14 +2842,16 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory:
@@ -3188,6 +3192,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7f8, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
@@ -3213,8 +3218,9 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[8:9], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3234,11 +3240,13 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[2:3]
+; GFX12-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -3596,6 +3604,7 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
@@ -3621,8 +3630,9 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[8:9], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[0:1], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3642,11 +3652,13 @@ define double @flat_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[2:3]
+; GFX12-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4005,6 +4017,7 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4020,18 +4033,20 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4040,17 +4055,19 @@ define void @flat_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB21_2
; GFX12-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[6:7]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4386,13 +4403,14 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, 0x7f8, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB22_3
; GFX12-NEXT: ; %bb.1: ; %Flow2
@@ -4404,18 +4422,20 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4423,18 +4443,20 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execnz .LBB22_4
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB22_2
; GFX12-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4786,13 +4808,14 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, 0xfffff800, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB23_3
; GFX12-NEXT: ; %bb.1: ; %Flow2
@@ -4804,18 +4827,20 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4823,18 +4848,20 @@ define void @flat_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execnz .LBB23_4
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB23_2
; GFX12-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -5187,27 +5214,29 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB24_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5216,7 +5245,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB24_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
@@ -5225,14 +5254,16 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory:
@@ -5607,27 +5638,29 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB25_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5636,7 +5669,7 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB25_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
@@ -5645,14 +5678,16 @@ define double @flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -5998,8 +6033,9 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6013,11 +6049,12 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6042,8 +6079,9 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6057,11 +6095,12 @@ define half @flat_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6382,34 +6421,36 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6417,7 +6458,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6430,13 +6471,15 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6444,11 +6487,12 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6512,40 +6556,38 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6781,34 +6823,36 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6816,7 +6860,7 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6829,13 +6873,15 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6843,11 +6889,12 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6912,40 +6959,38 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -7182,8 +7227,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7196,10 +7242,12 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7224,8 +7272,9 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7238,10 +7287,12 @@ define void @flat_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7554,33 +7605,36 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7597,33 +7651,37 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7681,31 +7739,29 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7941,33 +7997,36 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7984,33 +8043,37 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8069,31 +8132,29 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8330,15 +8391,17 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8363,15 +8426,17 @@ define half @flat_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8627,13 +8692,16 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8657,21 +8725,24 @@ define void @flat_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v3, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8914,35 +8985,37 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8950,7 +9023,7 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8963,13 +9036,15 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8977,11 +9052,12 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
@@ -9046,40 +9122,38 @@ define half @flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9318,34 +9392,37 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9362,34 +9439,38 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9447,31 +9528,29 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9707,58 +9786,114 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9803,54 +9938,106 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10051,61 +10238,119 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10152,56 +10397,110 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10408,61 +10707,119 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10510,56 +10867,110 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10761,61 +11172,116 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 -1024
- %result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
- ret bfloat %result
- }
-
-define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret bfloat %result
+ }
+
+define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10859,52 +11325,103 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11802,32 +12319,33 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11835,7 +12353,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11920,40 +12438,41 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12417,89 +12936,148 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
-; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v3, v2
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB43_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
- %unused = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
- ret void
-}
-
-define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: v_mov_b32_e32 v3, v2
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB43_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
+ %unused = atomicrmw fmax ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12546,56 +13124,110 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -13168,13 +13800,15 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13397,13 +14031,15 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13631,13 +14267,15 @@ define <2 x half> @flat_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13879,18 +14517,21 @@ define void @flat_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14097,18 +14738,21 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14322,18 +14966,21 @@ define void @flat_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14567,13 +15214,15 @@ define <2 x half> @flat_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -14804,19 +15453,22 @@ define void @flat_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15046,12 +15698,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15063,10 +15715,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -15183,18 +15838,19 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -15205,9 +15861,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15219,6 +15878,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -15496,12 +16156,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15513,10 +16173,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -15633,18 +16296,19 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -15655,9 +16319,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15669,6 +16336,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -15951,12 +16619,12 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15968,10 +16636,13 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16096,17 +16767,19 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_dual_max_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v0, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -16118,8 +16791,11 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16131,6 +16807,7 @@ define <2 x bfloat> @flat_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -16420,11 +17097,11 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16434,12 +17111,15 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16554,15 +17234,16 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16571,12 +17252,15 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16589,6 +17273,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -16853,11 +17538,11 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16867,12 +17552,15 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16987,15 +17675,16 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17004,12 +17693,15 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17022,6 +17714,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17293,11 +17986,11 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17307,12 +18000,15 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -17436,15 +18132,16 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17453,12 +18150,15 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17471,6 +18171,7 @@ define void @flat_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17757,12 +18458,12 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -17774,10 +18475,13 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -17896,18 +18600,19 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -17918,9 +18623,12 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17932,6 +18640,7 @@ define <2 x bfloat> @flat_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -18215,11 +18924,11 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -18229,12 +18938,15 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -18351,15 +19063,16 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -18368,12 +19081,15 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18386,6 +19102,7 @@ define void @flat_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index 9d8629b0b1d28..def9fe8c0f6b4 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -2802,27 +2802,29 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB18_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB18_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -2831,7 +2833,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB18_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
@@ -2840,14 +2842,16 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB18_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory:
@@ -3188,6 +3192,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0x7f8, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v5, null, 0, v1, vcc_lo
@@ -3213,8 +3218,9 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[8:9], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[0:1], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3234,11 +3240,13 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: .LBB19_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[4:5], v[2:3]
+; GFX12-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -3596,6 +3604,7 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v5, null, -1, v1, vcc_lo
@@ -3621,8 +3630,9 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[8:9], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[0:1], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[4:5], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -3642,11 +3652,13 @@ define double @flat_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: .LBB20_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[4:5]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v4, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v4, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[2:3], off
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[4:5], v[2:3]
+; GFX12-NEXT: scratch_store_b64 v6, v[2:3], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4005,6 +4017,7 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4020,18 +4033,20 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB21_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB21_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4040,17 +4055,19 @@ define void @flat_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(ptr
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB21_2
; GFX12-NEXT: .LBB21_6: ; %atomicrmw.private
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[6:7]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4386,13 +4403,14 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, 0x7f8, v0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, 0x7f8, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, 0, v1, vcc_lo
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB22_3
; GFX12-NEXT: ; %bb.1: ; %Flow2
@@ -4404,18 +4422,20 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB22_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB22_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4423,18 +4443,20 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execnz .LBB22_4
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB22_2
; GFX12-NEXT: .LBB22_6: ; %atomicrmw.private
-; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -4786,13 +4808,14 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, 0xfffff800, v0
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, -1, v1, vcc_lo
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
+; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v7
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execnz .LBB23_3
; GFX12-NEXT: ; %bb.1: ; %Flow2
@@ -4804,18 +4827,20 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX12-NEXT: .LBB23_3: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[6:7], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[6:7]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB23_4: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[0:1], v[6:7], v[0:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[0:1], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4823,18 +4848,20 @@ define void @flat_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_gra
; GFX12-NEXT: s_cbranch_execnz .LBB23_4
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr6_vgpr7
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX12-NEXT: s_cbranch_execz .LBB23_2
; GFX12-NEXT: .LBB23_6: ; %atomicrmw.private
-; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
+; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[6:7]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v4, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[0:1], v4, off
+; GFX12-NEXT: v_cndmask_b32_e32 v2, -1, v6, vcc_lo
+; GFX12-NEXT: scratch_load_b64 v[0:1], v2, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: scratch_store_b64 v4, v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
+; GFX12-NEXT: scratch_store_b64 v2, v[0:1], off
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
@@ -5187,27 +5214,29 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB24_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB24_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5216,7 +5245,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB24_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
@@ -5225,14 +5254,16 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr %ptr,
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB24_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory:
@@ -5607,27 +5638,29 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b64 s[0:1], src_private_base
; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_cmpx_ne_u32_e32 s1, v1
; GFX12-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX12-NEXT: s_cbranch_execz .LBB25_4
; GFX12-NEXT: ; %bb.1: ; %atomicrmw.global
-; GFX12-NEXT: flat_load_b64 v[4:5], v[0:1]
+; GFX12-NEXT: flat_load_b64 v[2:3], v[0:1]
; GFX12-NEXT: s_mov_b32 s1, 0
; GFX12-NEXT: .LBB25_2: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_dual_mov_b32 v9, v3 :: v_dual_mov_b32 v8, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[8:9], v[8:9]
+; GFX12-NEXT: v_min_num_f64_e32 v[6:7], v[2:3], v[4:5]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b64 v[2:3], v[0:1], v[6:9] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[8:9]
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s1, vcc_lo, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5636,7 +5669,7 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: ; %bb.3: ; %Flow
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
; GFX12-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX12-NEXT: ; implicit-def: $vgpr2_vgpr3
+; GFX12-NEXT: ; implicit-def: $vgpr4_vgpr5
; GFX12-NEXT: .LBB25_4: ; %Flow2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_not1_saveexec_b32 s0, s0
@@ -5645,14 +5678,16 @@ define double @flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__am
; GFX12-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[0:1]
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_cndmask_b32_e32 v6, -1, v0, vcc_lo
-; GFX12-NEXT: scratch_load_b64 v[4:5], v6, off
+; GFX12-NEXT: scratch_load_b64 v[2:3], v6, off
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[4:5], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: scratch_store_b64 v6, v[0:1], off
; GFX12-NEXT: .LBB25_6: ; %atomicrmw.phi
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX12-NEXT: v_dual_mov_b32 v0, v2 :: v_dual_mov_b32 v1, v3
; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__amdgpu_no_remote_memory:
@@ -5998,8 +6033,9 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6013,11 +6049,12 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6042,8 +6079,9 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
@@ -6057,11 +6095,12 @@ define half @flat_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr %
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6382,34 +6421,36 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6417,7 +6458,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6430,13 +6471,15 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6444,11 +6487,12 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6512,40 +6556,38 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -6781,34 +6823,36 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6816,7 +6860,7 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -6829,13 +6873,15 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -6843,11 +6889,12 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6912,40 +6959,38 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grain
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -7182,8 +7227,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7196,10 +7242,12 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7224,8 +7272,9 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
@@ -7238,10 +7287,12 @@ define void @flat_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7554,33 +7605,36 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7597,33 +7651,37 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7681,31 +7739,29 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7941,33 +7997,36 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7984,33 +8043,37 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8069,31 +8132,29 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8330,15 +8391,17 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8363,15 +8426,17 @@ define half @flat_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_fi
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v4, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v4
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8627,13 +8692,16 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -8657,21 +8725,24 @@ define void @flat_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_no_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v4, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v3, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8914,35 +8985,37 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8950,7 +9023,7 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8963,13 +9036,15 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8977,11 +9052,12 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
@@ -9046,40 +9122,38 @@ define half @flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grai
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[3:4]
+; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v6, v[3:4], v[6:7] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -9318,34 +9392,37 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9362,34 +9439,38 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -9447,31 +9528,29 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: flat_load_b32 v6, v[3:4]
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[3:4], v[5:6] glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -9707,58 +9786,114 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9803,54 +9938,106 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10051,61 +10238,119 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10152,56 +10397,110 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10408,61 +10707,119 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10510,56 +10867,110 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10761,61 +11172,116 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX7-NEXT: v_lshrrev_b32_e32 v0, v2, v5
; GFX7-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr inbounds bfloat, ptr %ptr, i64 -1024
- %result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
- ret bfloat %result
- }
-
-define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+ %result = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
+ ret bfloat %result
+ }
+
+define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10859,52 +11325,103 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11802,32 +12319,33 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11835,7 +12353,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11920,40 +12438,41 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12417,89 +12936,148 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
-; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX7-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
-; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX7-NEXT: buffer_wbinvl1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; GFX7-NEXT: v_mov_b32_e32 v3, v2
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_cbranch_execnz .LBB43_1
-; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
-; GFX7-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
- %unused = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
- ret void
-}
-
-define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX7-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX7-NEXT: .LBB43_1: ; %atomicrmw.start
+; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX7-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX7-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX7-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX7-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GFX7-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX7-NEXT: buffer_wbinvl1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX7-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX7-NEXT: v_mov_b32_e32 v3, v2
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_cbranch_execnz .LBB43_1
+; GFX7-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX7-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr inbounds bfloat, ptr %ptr, i64 1023
+ %unused = atomicrmw fmin ptr %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
+; GFX12-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12546,56 +13124,110 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -13168,13 +13800,15 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_memo
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13397,13 +14031,15 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_fi
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13631,13 +14267,15 @@ define <2 x half> @flat_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_fi
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -13879,18 +14517,21 @@ define void @flat_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14097,18 +14738,21 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14322,18 +14966,21 @@ define void @flat_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:-2048
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14567,13 +15214,15 @@ define <2 x half> @flat_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_f
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -14804,19 +15453,22 @@ define void @flat_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine_
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: flat_load_b32 v4, v[0:1] offset:2044
+; GFX12-NEXT: flat_load_b32 v3, v[0:1] offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -15046,12 +15698,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15063,10 +15715,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -15183,18 +15838,19 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -15205,9 +15861,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15219,6 +15878,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained_m
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -15496,12 +16156,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15513,10 +16173,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -15633,18 +16296,19 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -15655,9 +16319,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -15669,6 +16336,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -15951,12 +16619,12 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15968,10 +16636,13 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16096,17 +16767,19 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v0, v[3:4]
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_dual_min_f32 v5, v5, v1 :: v_dual_lshlrev_b32 v0, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -16118,8 +16791,11 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v0.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v0, v[3:4], v[5:6] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16131,6 +16807,7 @@ define <2 x bfloat> @flat_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_no
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -16420,11 +17097,11 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16434,12 +17111,15 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16554,15 +17234,16 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16571,12 +17252,15 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -16589,6 +17273,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memory(
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -16853,11 +17538,11 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16867,12 +17552,15 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -16987,15 +17675,16 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17004,12 +17693,15 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17022,6 +17714,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17293,11 +17986,11 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17307,12 +18000,15 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -17436,15 +18132,16 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17453,12 +18150,15 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17471,6 +18171,7 @@ define void @flat_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17757,12 +18458,12 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -17774,10 +18475,13 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -17896,18 +18600,19 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -17918,9 +18623,12 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[5:6] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -17932,6 +18640,7 @@ define <2 x bfloat> @flat_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_n
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -18215,11 +18924,11 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -18229,12 +18938,15 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -18351,15 +19063,16 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -18368,12 +19081,15 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -18386,6 +19102,7 @@ define void @flat_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fine
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index cc5204d00efdd..be2aa7d9f0a0d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9382,58 +9382,114 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16:
; GFX942: ; %bb.0:
@@ -9478,54 +9534,106 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16:
; GFX10: ; %bb.0:
@@ -9725,61 +9833,119 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
}
define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -9826,56 +9992,110 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
@@ -10081,61 +10301,119 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
}
define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX942: ; %bb.0:
@@ -10183,56 +10461,110 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX10: ; %bb.0:
@@ -10438,56 +10770,111 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
}
define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_noret_bf16:
; GFX942: ; %bb.0:
@@ -10531,52 +10918,103 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_noret_bf16:
; GFX10: ; %bb.0:
@@ -11471,32 +11909,33 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11504,7 +11943,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -11589,40 +12028,41 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -12111,62 +12551,121 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
}
define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -12213,56 +12712,110 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll b/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
index b64fb5022cdb8..75b939778d0d7 100644
--- a/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-sopc-vopc.ll
@@ -1314,24 +1314,28 @@ define <8 x i1> @vector_f32_ole() {
; GISEL-GFX1150-NEXT: v_readfirstlane_b32 s7, v7
; GISEL-GFX1150-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s1, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1150-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s2, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s3, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1150-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s4, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1150-NEXT: s_cselect_b32 s4, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s5, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v4.l, s4
; GISEL-GFX1150-NEXT: s_cselect_b32 s5, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s6, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v5.l, s5
; GISEL-GFX1150-NEXT: s_cselect_b32 s6, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_le_f32 s7, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v6.l, s6
; GISEL-GFX1150-NEXT: s_cselect_b32 s7, 1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v7.l, s7
; GISEL-GFX1150-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX1200-LABEL: vector_f32_ole:
@@ -1399,27 +1403,35 @@ define <8 x i1> @vector_f32_ole() {
; GISEL-GFX1200-NEXT: v_readfirstlane_b32 s7, v7
; GISEL-GFX1200-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s1, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1200-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s2, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s3, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1200-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s4, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1200-NEXT: s_cselect_b32 s4, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s5, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v4.l, s4
; GISEL-GFX1200-NEXT: s_cselect_b32 s5, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s6, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v5.l, s5
; GISEL-GFX1200-NEXT: s_cselect_b32 s6, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_le_f32 s7, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v6.l, s6
; GISEL-GFX1200-NEXT: s_cselect_b32 s7, 1, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v7.l, s7
; GISEL-GFX1200-NEXT: s_setpc_b64 s[30:31]
entry:
%LGV = load <8 x float>, ptr addrspace(1) null, align 32
@@ -1457,14 +1469,16 @@ define <4 x i1> @vector_f32_ogt() {
; GISEL-GFX1150-NEXT: s_cmp_gt_f32 s0, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_gt_f32 s1, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1150-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_gt_f32 s2, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_gt_f32 s3, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1150-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1150-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX1200-LABEL: vector_f32_ogt:
@@ -1508,15 +1522,19 @@ define <4 x i1> @vector_f32_ogt() {
; GISEL-GFX1200-NEXT: s_cmp_gt_f32 s0, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_gt_f32 s1, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1200-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_gt_f32 s2, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_gt_f32 s3, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1200-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1200-NEXT: s_setpc_b64 s[30:31]
entry:
%LGV = load <4 x float>, ptr addrspace(1) null, align 16
@@ -1538,6 +1556,7 @@ define <32 x i1> @vector_f32_ueq() {
; SDAG-GFX1150-NEXT: global_load_b128 v[4:7], v[4:5], off
; SDAG-GFX1150-NEXT: v_dual_mov_b32 v12, 48 :: v_dual_mov_b32 v13, 0
; SDAG-GFX1150-NEXT: v_dual_mov_b32 v16, 32 :: v_dual_mov_b32 v17, 0
+; SDAG-GFX1150-NEXT: s_clause 0x2
; SDAG-GFX1150-NEXT: global_load_b128 v[8:11], v[8:9], off
; SDAG-GFX1150-NEXT: global_load_b128 v[12:15], v[12:13], off
; SDAG-GFX1150-NEXT: global_load_b128 v[32:35], v[16:17], off
@@ -1699,68 +1718,84 @@ define <32 x i1> @vector_f32_ueq() {
; GISEL-GFX1150-NEXT: v_readfirstlane_b32 s41, v31
; GISEL-GFX1150-NEXT: s_cselect_b32 s8, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s9, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v1.l, s1
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1150-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s10, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v3.l, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v4.l, s4
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v5.l, s5
; GISEL-GFX1150-NEXT: s_cselect_b32 s10, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s11, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v6.l, s6
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v7.l, s7
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v8.l, s8
; GISEL-GFX1150-NEXT: s_cselect_b32 s11, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s12, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v9.l, s9
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v10.l, s10
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v11.l, s11
; GISEL-GFX1150-NEXT: s_cselect_b32 s12, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s13, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v12.l, s12
; GISEL-GFX1150-NEXT: s_cselect_b32 s13, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s14, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v13.l, s13
; GISEL-GFX1150-NEXT: s_cselect_b32 s14, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s15, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v14.l, s14
; GISEL-GFX1150-NEXT: s_cselect_b32 s15, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s16, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v15.l, s15
; GISEL-GFX1150-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s17, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v16.l, s16
; GISEL-GFX1150-NEXT: s_cselect_b32 s17, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s18, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v17.l, s17
; GISEL-GFX1150-NEXT: s_cselect_b32 s18, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s19, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v18.l, s18
; GISEL-GFX1150-NEXT: s_cselect_b32 s19, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s20, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v19.l, s19
; GISEL-GFX1150-NEXT: s_cselect_b32 s20, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s21, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v20.l, s20
; GISEL-GFX1150-NEXT: s_cselect_b32 s21, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s22, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v21.l, s21
; GISEL-GFX1150-NEXT: s_cselect_b32 s22, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s23, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v22.l, s22
; GISEL-GFX1150-NEXT: s_cselect_b32 s23, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s24, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v23.l, s23
; GISEL-GFX1150-NEXT: s_cselect_b32 s24, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s25, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v24.l, s24
; GISEL-GFX1150-NEXT: s_cselect_b32 s25, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s26, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v25.l, s25
; GISEL-GFX1150-NEXT: s_cselect_b32 s26, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s27, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v26.l, s26
; GISEL-GFX1150-NEXT: s_cselect_b32 s27, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s28, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v27.l, s27
; GISEL-GFX1150-NEXT: s_cselect_b32 s28, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s29, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v28.l, s28
; GISEL-GFX1150-NEXT: s_cselect_b32 s29, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s40, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v29.l, s29
; GISEL-GFX1150-NEXT: s_cselect_b32 s40, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nlg_f32 s41, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v30.l, s40
; GISEL-GFX1150-NEXT: s_cselect_b32 s41, 1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v30, s40 :: v_dual_mov_b32 v31, s41
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v31.l, s41
; GISEL-GFX1150-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX1200-LABEL: vector_f32_ueq:
@@ -1780,6 +1815,7 @@ define <32 x i1> @vector_f32_ueq() {
; SDAG-GFX1200-NEXT: global_load_b128 v[4:7], v[4:5], off
; SDAG-GFX1200-NEXT: v_dual_mov_b32 v12, 48 :: v_dual_mov_b32 v13, 0
; SDAG-GFX1200-NEXT: v_dual_mov_b32 v16, 32 :: v_dual_mov_b32 v17, 0
+; SDAG-GFX1200-NEXT: s_clause 0x2
; SDAG-GFX1200-NEXT: global_load_b128 v[8:11], v[8:9], off
; SDAG-GFX1200-NEXT: global_load_b128 v[12:15], v[12:13], off
; SDAG-GFX1200-NEXT: global_load_b128 v[32:35], v[16:17], off
@@ -1978,79 +2014,104 @@ define <32 x i1> @vector_f32_ueq() {
; GISEL-GFX1200-NEXT: s_cselect_b32 s8, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s9, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v0.l, s0
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v1.l, s1
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1200-NEXT: s_cselect_b32 s9, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s10, 0
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v3.l, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v4.l, s4
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v5.l, s5
; GISEL-GFX1200-NEXT: s_cselect_b32 s10, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s11, 0
-; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v9, s9
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v6.l, s6
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v7.l, s7
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v8.l, s8
; GISEL-GFX1200-NEXT: s_cselect_b32 s11, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s12, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v10, s10 :: v_dual_mov_b32 v11, s11
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v9.l, s9
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v10.l, s10
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v11.l, s11
; GISEL-GFX1200-NEXT: s_cselect_b32 s12, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s13, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v12.l, s12
; GISEL-GFX1200-NEXT: s_cselect_b32 s13, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s14, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v12, s12 :: v_dual_mov_b32 v13, s13
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v13.l, s13
; GISEL-GFX1200-NEXT: s_cselect_b32 s14, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s15, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v14.l, s14
; GISEL-GFX1200-NEXT: s_cselect_b32 s15, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s16, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v15.l, s15
; GISEL-GFX1200-NEXT: s_cselect_b32 s16, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s17, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v16.l, s16
; GISEL-GFX1200-NEXT: s_cselect_b32 s17, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s18, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s17
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v17.l, s17
; GISEL-GFX1200-NEXT: s_cselect_b32 s18, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s19, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v18.l, s18
; GISEL-GFX1200-NEXT: s_cselect_b32 s19, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s20, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v18, s18 :: v_dual_mov_b32 v19, s19
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v19.l, s19
; GISEL-GFX1200-NEXT: s_cselect_b32 s20, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s21, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v20.l, s20
; GISEL-GFX1200-NEXT: s_cselect_b32 s21, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s22, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v20, s20 :: v_dual_mov_b32 v21, s21
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v21.l, s21
; GISEL-GFX1200-NEXT: s_cselect_b32 s22, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s23, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v22.l, s22
; GISEL-GFX1200-NEXT: s_cselect_b32 s23, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s24, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v22, s22 :: v_dual_mov_b32 v23, s23
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v23.l, s23
; GISEL-GFX1200-NEXT: s_cselect_b32 s24, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s25, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v24.l, s24
; GISEL-GFX1200-NEXT: s_cselect_b32 s25, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s26, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v24, s24 :: v_dual_mov_b32 v25, s25
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v25.l, s25
; GISEL-GFX1200-NEXT: s_cselect_b32 s26, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s27, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v26.l, s26
; GISEL-GFX1200-NEXT: s_cselect_b32 s27, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s28, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v27, s27
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v27.l, s27
; GISEL-GFX1200-NEXT: s_cselect_b32 s28, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s29, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v28.l, s28
; GISEL-GFX1200-NEXT: s_cselect_b32 s29, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s40, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v28, s28 :: v_dual_mov_b32 v29, s29
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v29.l, s29
; GISEL-GFX1200-NEXT: s_cselect_b32 s40, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nlg_f32 s41, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v30.l, s40
; GISEL-GFX1200-NEXT: s_cselect_b32 s41, 1, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v30, s40 :: v_dual_mov_b32 v31, s41
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v31.l, s41
; GISEL-GFX1200-NEXT: s_setpc_b64 s[30:31]
entry:
%LGV = load <32 x float>, ptr addrspace(1) null, align 128
@@ -2088,14 +2149,16 @@ define <4 x i1> @vector_f32_ugt() {
; GISEL-GFX1150-NEXT: s_cmp_nle_f32 s0, 0
; GISEL-GFX1150-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nle_f32 s1, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1150-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nle_f32 s2, 0
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1150-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1150-NEXT: s_cmp_nle_f32 s3, 0
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1150-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GISEL-GFX1150-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1150-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1150-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX1200-LABEL: vector_f32_ugt:
@@ -2139,15 +2202,19 @@ define <4 x i1> @vector_f32_ugt() {
; GISEL-GFX1200-NEXT: s_cmp_nle_f32 s0, 0
; GISEL-GFX1200-NEXT: s_cselect_b32 s0, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nle_f32 s1, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX1200-NEXT: s_cselect_b32 s1, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nle_f32 s2, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v1.l, s1
; GISEL-GFX1200-NEXT: s_cselect_b32 s2, 1, 0
; GISEL-GFX1200-NEXT: s_cmp_nle_f32 s3, 0
+; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v2.l, s2
; GISEL-GFX1200-NEXT: s_cselect_b32 s3, 1, 0
; GISEL-GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GISEL-GFX1200-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GISEL-GFX1200-NEXT: v_mov_b16_e32 v3.l, s3
; GISEL-GFX1200-NEXT: s_setpc_b64 s[30:31]
entry:
%LGV1 = load <4 x float>, ptr addrspace(1) null, align 16
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
index 982bd74b68635..25ea830cf3c2d 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-f16-hw.ll
@@ -518,11 +518,11 @@ define <3 x i8> @to_fp8_v3f16(<3 x half> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.l, v0
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.h, v1
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.l, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -780,11 +780,11 @@ define <3 x i8> @to_bf8_v3f16(<3 x half> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.l, v0
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.h, v1
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.l, v0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -1082,15 +1082,13 @@ define <4 x i8> @to_fp8_v4f16(<4 x half> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.h, v1
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.l, v0
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v2.l, v1
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v0.l, v0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v0, 16, v1 :: v_dual_lshrrev_b32 v1, 8, v2
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v3, 16, v2 :: v_dual_lshrrev_b32 v1, 8, v0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: to_fp8_v4f16:
@@ -1350,15 +1348,13 @@ define <4 x i8> @to_bf8_v4f16(<4 x half> %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.h, v1
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.l, v0
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v2.l, v1
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf8_f16_e64 v0.l, v0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v0, 16, v1 :: v_dual_lshrrev_b32 v1, 8, v2
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX1250-TRUE16-NEXT: v_dual_lshlrev_b32 v3, 16, v2 :: v_dual_lshrrev_b32 v1, 8, v0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: to_bf8_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
index cc038e9cb65ea..bad68376c242e 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
@@ -1596,81 +1596,86 @@ define i8 @to_fp8_bf16(bfloat %x) {
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX1200-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1200-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_frexp_mant_f32_e32 v2, v5
-; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v5
-; GFX1200-NEXT: v_and_b16 v0.l, 0x80, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_bfe_u32 v3, v2, 16, 1
-; GFX1200-NEXT: v_xor_b16 v1.l, v6.l, -1
-; GFX1200-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX1200-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX1200-NEXT: v_min_u16 v1.l, v1.l, 15
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc_lo
-; GFX1200-NEXT: v_sub_nc_u16 v1.h, v1.l, 1 clamp
+; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX1200-NEXT: v_and_b16 v0.l, 0x80, v0.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_frexp_mant_f32_e32 v1, v5
+; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v5
+; GFX1200-NEXT: v_bfe_u32 v2, v1, 16, 1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_xor_b16 v0.h, v6.l, -1
+; GFX1200-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX1200-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX1200-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_min_u16 v0.h, v0.h, 15
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc_lo
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_and_b16 v2.l, 0x7f, v7.h
-; GFX1200-NEXT: v_lshlrev_b16 v2.h, v1.h, 1
-; GFX1200-NEXT: v_and_b16 v3.l, v7.h, 7
+; GFX1200-NEXT: v_sub_nc_u16 v1.l, v0.h, 1 clamp
+; GFX1200-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_lshlrev_b16 v1.h, v1.l, 1
; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 19, v7
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v3.h, 0x80, v2.l
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, -1
+; GFX1200-NEXT: v_and_b16 v2.l, 0x7f, v3.l
+; GFX1200-NEXT: v_and_b16 v2.h, v3.l, 7
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, -1
+; GFX1200-NEXT: v_or_b16 v3.l, 0x80, v2.l
; GFX1200-NEXT: v_lshrrev_b16 v2.l, 4, v2.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.h
-; GFX1200-NEXT: v_and_b16 v2.h, v3.h, v2.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_and_b16 v3.l, v2.l, 1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
+; GFX1200-NEXT: v_and_b16 v1.h, v3.l, v1.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_and_b16 v2.h, v2.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, v1.l, v3.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
-; GFX1200-NEXT: v_or_b16 v2.h, v4.l, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_and_b16 v3.l, v4.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v3.h, v0.h, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v1.l, v1.l, v3.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_or_b16 v1.h, v4.l, v2.h
+; GFX1200-NEXT: v_and_b16 v2.h, v3.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
-; GFX1200-NEXT: v_and_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.h
+; GFX1200-NEXT: v_and_b16 v1.h, v7.l, v1.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_mov_b16_e32 v4.l, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v2.l, v1.h
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, v2.h
-; GFX1200-NEXT: v_or_b16 v3.l, v4.l, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
-; GFX1200-NEXT: v_and_b16 v1.l, v1.h, v3.l
+; GFX1200-NEXT: v_or_b16 v2.h, v4.l, v2.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v1.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_and_b16 v0.h, v1.l, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
+; GFX1200-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s0
; GFX1200-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v0.h, v4.h, v1.l
-; GFX1200-NEXT: v_add_nc_u16 v1.l, v6.l, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v0.h, 0, v0.h, vcc_lo
+; GFX1200-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_add_nc_u16 v0.h, v3.h, v0.h
+; GFX1200-NEXT: v_add_nc_u16 v1.l, v6.l, v1.l
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v0.h
; GFX1200-NEXT: v_add_nc_u16 v1.l, v1.l, 6
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, 8, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_lshlrev_b16 v2.h, 3, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, 8, vcc_lo
; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_lshlrev_b16 v2.h, 3, v1.l
; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l
-; GFX1200-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_or_b16 v2.l, v0.l, v2.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_or_b16 v2.h, v0.l, v2.h
-; GFX1200-NEXT: v_or_b16 v0.h, v1.h, v0.h
+; GFX1200-NEXT: v_or_b16 v0.h, v2.l, v0.h
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1200-NEXT: v_or_b16 v1.l, v2.h, v2.l
+; GFX1200-NEXT: v_or_b16 v1.l, v2.h, v1.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo
; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v5, v5
@@ -1685,73 +1690,74 @@ define i8 @to_fp8_bf16(bfloat %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0x80
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0x80
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v7, v6
-; GFX1250-TRUE16-NEXT: v_frexp_mant_f32_e32 v2, v6
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX1250-TRUE16-NEXT: v_and_b16 v0.l, 0x80, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_xor_b16 v1.l, v7.l, -1
-; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v2, v2, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v7, v6
+; GFX1250-TRUE16-NEXT: v_frexp_mant_f32_e32 v1, v6
+; GFX1250-TRUE16-NEXT: v_xor_b16 v0.h, v7.l, -1
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 15
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, 0x7f, v2.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v3.l, v2.l, 7
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.h, v1.l, 1 clamp
-; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v4.h, 4, v2.h
+; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
+; GFX1250-TRUE16-NEXT: v_min_u16 v0.h, v0.h, 15
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, 0x7f, v1.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v1.l, 7
+; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.h, v0.h, 1 clamp
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.h, v1.h, 1
+; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v4.l, 4, v2.l
+; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, v1.h, 1
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v3.l, v3.h, -1
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.h, v2.l, v4.l, 0x7f bitop3:0xec
-; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.l, 3, v2.l
-; GFX1250-TRUE16-NEXT: v_and_b16 v4.l, v4.h, 1
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.h, v2.h, v3.l, 0x80 bitop3:0xc8
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v3.l, v1.l, v3.h
-; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.h
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v2.l, v5.l, v4.l bitop3:0xe0
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, -1
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v1.l, v3.h, 0x7f bitop3:0xec
+; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.l, 3, v1.l
+; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v4.l, 1
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
-; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v3.l, 1
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.l, v4.h, v2.l
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v2.l, v2.h, 0x80 bitop3:0xc8
+; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.h, v0.h, v3.l
+; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v1.l, v5.l, v3.h bitop3:0xe0
+; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.h, 1
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, v1.l
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
-; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v1.h, v4.l, v2.h bitop3:0xe0
-; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.h, v2.l, 0, s0
+; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v1.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s0
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v3.l, v2.l bitop3:0xe0
; GFX1250-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v7.l, v4.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l
+; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, v0.h, vcc_lo
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v3.l, v1.l
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, 6
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, v2.l
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.h, v0.h
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v1.l
-; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v0.h
-; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, vcc_lo
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.h, 6
+; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v0.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v1.h
+; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.h, 0, 8, vcc_lo
-; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v0.h
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.l, v1.h, v2.l bitop3:0xfe
-; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v1.l, v2.h bitop3:0xfe
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, v1.l, vcc_lo
+; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v1.l, v2.l bitop3:0xfe
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.l, v0.h, v2.h bitop3:0xfe
+; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v6
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, vcc_lo
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
index 9b9eecf4dee0e..4568866830a0c 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-widen.ll
@@ -610,372 +610,373 @@ define <5 x i8> @to_fp8_v5f16(<5 x half> %x) {
; GFX1200-NEXT: s_wait_kmcnt 0x0
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v2.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v2.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v2.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v2.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v1.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.h, v1.l
; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v3.l
-; GFX1200-NEXT: v_and_b16 v6.h, 0x3ff, v3.h
-; GFX1200-NEXT: v_and_b16 v7.l, v3.h, 63
-; GFX1200-NEXT: v_sub_nc_u16 v8.l, 2, v5.h
+; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v3.h
+; GFX1200-NEXT: v_and_b16 v6.l, v3.h, 63
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v13.l
; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h
; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
-; GFX1200-NEXT: v_lshrrev_b16 v10.l, 7, v6.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
-; GFX1200-NEXT: v_or_b16 v6.h, 0x400, v6.h
-; GFX1200-NEXT: v_min_u16 v7.l, v8.l, 15
+; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v5.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l
+; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v7.l
; GFX1200-NEXT: v_sub_nc_u16 v8.h, v5.l, 1 clamp
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v4.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v13.l
+; GFX1200-NEXT: v_and_b16 v11.l, v5.h, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX1200-NEXT: v_lshrrev_b16 v10.h, v5.l, v8.l
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.h, 1
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
+; GFX1200-NEXT: v_min_u16 v5.l, v9.h, 15
+; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.l
+; GFX1200-NEXT: v_and_b16 v10.l, v6.h, 63
+; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
+; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v6.h
+; GFX1200-NEXT: v_sub_nc_u16 v11.h, v5.l, 1 clamp
+; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v11.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l
+; GFX1200-NEXT: v_and_b16 v9.l, v8.l, v9.l
+; GFX1200-NEXT: v_or_b16 v9.h, 0x400, v6.l
+; GFX1200-NEXT: v_lshlrev_b16 v11.l, v11.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v5.h, v3.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v8.h, v8.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l
+; GFX1200-NEXT: v_and_b16 v9.l, v10.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, v5.l, v9.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 63
-; GFX1200-NEXT: v_sub_nc_u16 v12.l, v7.l, 1 clamp
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, v8.h, 1
-; GFX1200-NEXT: v_or_b16 v11.h, 0x400, v7.h
-; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v6.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_lshlrev_b16 v12.h, v12.l, 1
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v10.h, -1
-; GFX1200-NEXT: v_and_b16 v10.h, v10.l, 1
-; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
-; GFX1200-NEXT: v_and_b16 v8.l, v6.h, v8.l
-; GFX1200-NEXT: v_or_b16 v10.h, v11.l, v10.h
-; GFX1200-NEXT: v_lshrrev_b16 v11.l, v5.l, v6.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v10.l
; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v10.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v8.l, v11.l, 1
-; GFX1200-NEXT: v_add_nc_u16 v10.h, v12.h, -1
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.l, v3.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX1200-NEXT: v_sub_nc_u16 v7.h, 2, v4.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v8.h, v6.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v8.l
-; GFX1200-NEXT: v_and_b16 v8.l, v11.h, v10.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, v7.l, v11.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.h
+; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v9.l
+; GFX1200-NEXT: v_add_nc_u16 v9.l, v11.l, -1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
-; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v6.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v8.l
-; GFX1200-NEXT: v_and_b16 v9.l, v8.h, 1
-; GFX1200-NEXT: v_min_u16 v6.h, v9.h, 15
-; GFX1200-NEXT: v_mov_b16_e32 v8.l, v10.l
-; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, s0
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s2
-; GFX1200-NEXT: v_lshrrev_b16 v9.h, v12.l, v11.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v7.l
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v8.l
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v11.l, v5.l
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
-; GFX1200-NEXT: v_and_b16 v8.l, v7.h, 1
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.l
-; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s0
-; GFX1200-NEXT: v_and_b16 v7.l, v9.h, v9.l
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v3.l
-; GFX1200-NEXT: v_or_b16 v8.l, v10.l, v8.l
-; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, s0
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s0
+; GFX1200-NEXT: v_and_b16 v5.l, v5.h, v8.h
+; GFX1200-NEXT: v_and_b16 v5.h, v9.h, v9.l
+; GFX1200-NEXT: v_and_b16 v8.h, v6.l, 1
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h
+; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v8.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v9.l
+; GFX1200-NEXT: v_and_b16 v8.h, v8.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v10.h, v5.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b16 v5.h, v6.h, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v11.h, v9.h
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, v5.h
; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, s2
-; GFX1200-NEXT: v_or_b16 v9.l, v4.l, v9.l
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v8.l
-; GFX1200-NEXT: v_or_b16 v9.h, v4.l, v9.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v2
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.h, v7.l
-; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
-; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v5.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.l, v1.l
-; GFX1200-NEXT: v_sub_nc_u16 v7.h, v6.h, 1 clamp
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 8, vcc_lo
+; GFX1200-NEXT: v_or_b16 v8.h, v9.l, v8.h
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
+; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l
+; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h
+; GFX1200-NEXT: v_or_b16 v8.h, v4.l, v9.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v5.l, v6.l, v5.l
+; GFX1200-NEXT: v_min_u16 v6.l, v7.h, 15
+; GFX1200-NEXT: v_or_b16 v3.h, v8.h, v3.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v6.h, s1
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, v9.l
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v6.l, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.l, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
-; GFX1200-NEXT: v_and_b16 v5.l, 0x3ff, v7.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, v7.h, 1
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, s1
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v1.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v2
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, s1
+; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v5.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.l
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, 6
+; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
+; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v4.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, 8, s0
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.l, v2.l
+; GFX1200-NEXT: v_lshlrev_b16 v2.l, 3, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0
+; GFX1200-NEXT: v_and_b16 v7.h, v8.l, v7.h
+; GFX1200-NEXT: v_or_b16 v2.h, v8.h, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v2.l, v8.h, v2.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v6.l, v8.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h
+; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v3.l
+; GFX1200-NEXT: v_and_b16 v3.l, v5.l, 63
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v5.h
+; GFX1200-NEXT: v_and_b16 v5.h, v9.l, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v7.l
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 6, v5.l
+; GFX1200-NEXT: v_or_b16 v3.l, v10.l, v5.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v8.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v2.h, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, 7, v10.l
-; GFX1200-NEXT: v_or_b16 v10.l, 0x400, v5.l
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s1
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v9.l
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l
-; GFX1200-NEXT: v_or_b16 v8.h, v9.h, v8.h
-; GFX1200-NEXT: v_and_b16 v4.l, v10.l, v8.l
-; GFX1200-NEXT: v_and_b16 v8.l, v7.l, 63
-; GFX1200-NEXT: v_add_nc_u16 v2.l, v5.h, 6
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v10.l
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
-; GFX1200-NEXT: v_lshlrev_b16 v4.l, 3, v2.l
-; GFX1200-NEXT: v_or_b16 v3.l, v8.h, v3.l
-; GFX1200-NEXT: v_and_b16 v8.h, v5.h, 1
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v2.h, v4.l, 1
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
+; GFX1200-NEXT: v_and_b16 v3.l, v5.h, v3.l
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v13.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v12.l
+; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v5.l
-; GFX1200-NEXT: v_or_b16 v4.l, v9.h, v4.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, v7.h, v10.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v6.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v4.l, v4.l, v4.h
-; GFX1200-NEXT: v_and_b16 v4.h, v5.l, 1
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v2.l
-; GFX1200-NEXT: v_or_b16 v8.l, v9.l, v8.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v7.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, vcc_lo
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
-; GFX1200-NEXT: v_and_b16 v2.l, v7.h, v8.l
-; GFX1200-NEXT: v_or_b16 v4.h, v9.l, v4.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.l, v1.h
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.h
-; GFX1200-NEXT: v_and_b16 v3.h, 0xff, v3.h
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, v2.l, s1
-; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s0
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v9.h, vcc_lo
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.h, v2.h
-; GFX1200-NEXT: v_add_nc_u16 v2.l, v5.h, v2.l
-; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v4.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.l, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v8.h, vcc_lo
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v13.l, v13.l
+; GFX1200-NEXT: v_and_b16 v2.h, v5.l, v2.h
+; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v9.l, v3.l
+; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v4.l, v2.h
+; GFX1200-NEXT: v_min_u16 v4.l, v5.l, 15
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.l
; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, s1
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.l, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
-; GFX1200-NEXT: v_min_u16 v2.h, v5.h, 15
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v1.h
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo
; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v2.h, 1 clamp
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s1
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v5.h
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
-; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0
; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, v8.l
; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_or_b16 v2.l, v7.l, v2.l
; GFX1200-NEXT: v_add_nc_u16 v8.h, v8.h, -1
-; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 63
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v2.h, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
+; GFX1200-NEXT: v_or_b16 v3.l, v7.l, v3.l
+; GFX1200-NEXT: v_and_b16 v7.l, v6.l, 63
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v8.l
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v6.l
-; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v4.h
; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l
+; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1
; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v2.h
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
-; GFX1200-NEXT: v_or_b16 v4.h, v7.l, v4.h
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
+; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
+; GFX1200-NEXT: v_and_b16 v3.h, 0xff, v3.h
; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v8.h
-; GFX1200-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
+; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v8.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v11, 16, v0
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v4.h, v2.l, vcc_lo
-; GFX1200-NEXT: v_and_b16 v6.l, v6.h, v9.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v3.l, vcc_lo
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.l
+; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v9.h
; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v8.l
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v7.h, v5.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v6.l, s0
-; GFX1200-NEXT: v_min_u16 v4.h, v6.h, 15
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v2.h
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v6.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v9.l, v5.h
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.h, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v2.h, v5.l, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.h
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v7.l
-; GFX1200-NEXT: v_or_b16 v5.l, 0x400, v7.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v7.l, v8.h, -1
-; GFX1200-NEXT: v_add_nc_u16 v1.l, v4.l, 6
+; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s0
+; GFX1200-NEXT: v_min_u16 v3.l, v6.h, 15
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l
+; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.l, v4.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v3.l, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v5.h, 0, s0
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v2.h, v6.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, v7.l
+; GFX1200-NEXT: v_or_b16 v5.h, 0x400, v7.h
; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v9.h, v4.h, v5.l
-; GFX1200-NEXT: v_and_b16 v5.h, v5.l, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v1.l
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v6.h, v5.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
-; GFX1200-NEXT: v_or_b16 v5.h, v8.h, v7.l
-; GFX1200-NEXT: v_or_b16 v7.l, v8.h, v9.l
-; GFX1200-NEXT: v_and_b16 v9.l, v9.h, 1
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, -1
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, 6
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v3.l, v5.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v1.l, v5.h, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v1.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v5.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v4.l, 0, s0
+; GFX1200-NEXT: v_or_b16 v4.h, v4.h, v2.h
+; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v7.l
+; GFX1200-NEXT: v_and_b16 v7.l, v9.l, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h
+; GFX1200-NEXT: v_or_b16 v2.h, v3.h, v2.l
+; GFX1200-NEXT: v_or_b16 v1.l, v4.l, v1.l
+; GFX1200-NEXT: v_sub_nc_u16 v4.l, 2, v9.h
+; GFX1200-NEXT: v_or_b16 v1.h, v10.l, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v5.l, s1
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v7.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, vcc_lo
+; GFX1200-NEXT: v_min_u16 v3.h, v4.l, 15
+; GFX1200-NEXT: v_and_b16 v4.l, v6.l, 63
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v11.l
+; GFX1200-NEXT: v_and_b16 v1.h, v5.h, v1.h
+; GFX1200-NEXT: v_sub_nc_u16 v4.h, v3.h, 1 clamp
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l
+; GFX1200-NEXT: v_and_b16 v6.h, v5.l, 1
+; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v3.l
+; GFX1200-NEXT: v_and_b16 v7.l, v3.l, 63
+; GFX1200-NEXT: v_lshlrev_b16 v5.h, v4.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v10.h, v0.h
-; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
-; GFX1200-NEXT: v_or_b16 v5.h, v5.h, v4.l
-; GFX1200-NEXT: v_or_b16 v4.l, v3.h, v3.l
-; GFX1200-NEXT: v_or_b16 v1.l, v10.l, v9.l
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, 2, v10.h
-; GFX1200-NEXT: v_and_b16 v3.h, v6.l, 63
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.h, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
-; GFX1200-NEXT: v_and_b16 v1.l, v5.l, v1.l
-; GFX1200-NEXT: v_min_u16 v3.l, v7.l, 15
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v0.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v7.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
-; GFX1200-NEXT: v_sub_nc_u16 v5.l, v3.l, 1 clamp
-; GFX1200-NEXT: v_and_b16 v3.h, 0x3ff, v4.h
-; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 63
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 1
-; GFX1200-NEXT: v_lshlrev_b16 v6.h, v5.l, 1
-; GFX1200-NEXT: v_or_b16 v10.l, 0x400, v3.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, 7, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, -1
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
+; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l
+; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v6.h
+; GFX1200-NEXT: v_lshrrev_b16 v10.h, v3.h, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v7.l, v9.l, v7.l
-; GFX1200-NEXT: v_and_b16 v7.h, v3.h, 1
-; GFX1200-NEXT: v_and_b16 v6.h, v10.l, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v5.l, v10.l
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v11.l, v3.l, v10.l
-; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v7.h
-; GFX1200-NEXT: v_and_b16 v7.l, v11.l, 1
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h
+; GFX1200-NEXT: v_and_b16 v7.l, v4.l, 1
+; GFX1200-NEXT: v_lshrrev_b16 v3.l, 6, v3.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
+; GFX1200-NEXT: v_and_b16 v5.h, v6.l, v6.h
+; GFX1200-NEXT: v_and_b16 v6.h, v10.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v7.h
+; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.l
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v6.h
-; GFX1200-NEXT: v_add_nc_u16 v1.l, v9.h, v1.l
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.h
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v4.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v9.l, v1.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.h
+; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v6.l
+; GFX1200-NEXT: v_mov_b16_e32 v7.l, v13.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.h
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v4.l, v3.l
+; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.l, v9.l, v7.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v0
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v8.h, s0
-; GFX1200-NEXT: v_and_b16 v4.h, v5.l, v6.l
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, s3
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v0.l
-; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s2
-; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s1
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.h, v6.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 7, v9.l
-; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v11.l, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s3
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.h, v1.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v3.l
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v6.l
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s2
-; GFX1200-NEXT: v_or_b16 v6.h, v5.l, v6.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s1
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
-; GFX1200-NEXT: v_or_b16 v9.l, v8.l, v9.l
-; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.l
-; GFX1200-NEXT: v_or_b16 v7.h, v8.l, v7.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v6.l
-; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7f, v2.h, s0
-; GFX1200-NEXT: v_or_b16 v1.l, v6.h, v1.l
-; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v4.h
-; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v5.h
-; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s2
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.l
+; GFX1200-NEXT: v_and_b16 v4.l, v4.h, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v8.l, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 8, s2
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v8.h, s0
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v4.l, s1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v0.l
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.h, v3.h
+; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v0
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
+; GFX1200-NEXT: v_or_b16 v0.h, v4.l, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, 6
+; GFX1200-NEXT: v_lshlrev_b16 v6.l, 3, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.h, v3.l, s1
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v0.h
-; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v5.h
+; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l
+; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v6.h
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h
+; GFX1200-NEXT: v_or_b16 v7.h, v7.l, v7.h
+; GFX1200-NEXT: v_or_b16 v1.h, v6.l, v5.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s0
+; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h
+; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v5.h, v1.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
-; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v4.l
-; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v0.h
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v11.l, v11.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.l, v3.h, s1
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v11.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
+; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v8.l, s1
-; GFX1200-NEXT: v_or_b32_e32 v6, 0x7f7f0000, v3
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v7.l, s1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v0.h, vcc_lo
+; GFX1200-NEXT: v_or_b32_e32 v4, 0x7f7f0000, v3
+; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0
-; GFX1200-NEXT: v_lshrrev_b64 v[5:6], 24, v[5:6]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4]
; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: s_setpc_b64 s[30:31]
@@ -1377,373 +1378,374 @@ define <6 x i8> @to_fp8_v6f16(<6 x half> %x) {
; GFX1200-NEXT: s_wait_kmcnt 0x0
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v2.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v2.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v2.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v2.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v1.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.h, v1.l
; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v3.l
-; GFX1200-NEXT: v_and_b16 v6.h, 0x3ff, v3.h
-; GFX1200-NEXT: v_and_b16 v7.l, v3.h, 63
-; GFX1200-NEXT: v_sub_nc_u16 v8.l, 2, v5.h
+; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v3.h
+; GFX1200-NEXT: v_and_b16 v6.l, v3.h, 63
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v13.l
; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h
; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
-; GFX1200-NEXT: v_lshrrev_b16 v10.l, 7, v6.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
-; GFX1200-NEXT: v_or_b16 v6.h, 0x400, v6.h
-; GFX1200-NEXT: v_min_u16 v7.l, v8.l, 15
+; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v5.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l
+; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v7.l
; GFX1200-NEXT: v_sub_nc_u16 v8.h, v5.l, 1 clamp
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v4.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v13.l
+; GFX1200-NEXT: v_and_b16 v11.l, v5.h, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX1200-NEXT: v_lshrrev_b16 v10.h, v5.l, v8.l
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.h, 1
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
+; GFX1200-NEXT: v_min_u16 v5.l, v9.h, 15
+; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.l
+; GFX1200-NEXT: v_and_b16 v10.l, v6.h, 63
+; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
+; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v6.h
+; GFX1200-NEXT: v_sub_nc_u16 v11.h, v5.l, 1 clamp
+; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v11.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l
+; GFX1200-NEXT: v_and_b16 v9.l, v8.l, v9.l
+; GFX1200-NEXT: v_or_b16 v9.h, 0x400, v6.l
+; GFX1200-NEXT: v_lshlrev_b16 v11.l, v11.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v5.h, v3.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v8.h, v8.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l
+; GFX1200-NEXT: v_and_b16 v9.l, v10.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, v5.l, v9.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_and_b16 v9.l, v4.h, 63
-; GFX1200-NEXT: v_sub_nc_u16 v12.l, v7.l, 1 clamp
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, v8.h, 1
-; GFX1200-NEXT: v_or_b16 v11.h, 0x400, v7.h
-; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v6.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_lshlrev_b16 v12.h, v12.l, 1
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v10.h, -1
-; GFX1200-NEXT: v_and_b16 v10.h, v10.l, 1
-; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
-; GFX1200-NEXT: v_and_b16 v8.l, v6.h, v8.l
-; GFX1200-NEXT: v_or_b16 v10.h, v11.l, v10.h
-; GFX1200-NEXT: v_lshrrev_b16 v11.l, v5.l, v6.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v10.l
; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v10.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v8.l, v11.l, 1
-; GFX1200-NEXT: v_add_nc_u16 v10.h, v12.h, -1
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.l, v3.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX1200-NEXT: v_sub_nc_u16 v7.h, 2, v4.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v8.h, v6.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v8.l
-; GFX1200-NEXT: v_and_b16 v8.l, v11.h, v10.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, v7.l, v11.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.h
+; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v9.l
+; GFX1200-NEXT: v_add_nc_u16 v9.l, v11.l, -1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
-; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v6.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v8.l
-; GFX1200-NEXT: v_and_b16 v9.l, v8.h, 1
-; GFX1200-NEXT: v_min_u16 v6.h, v9.h, 15
-; GFX1200-NEXT: v_mov_b16_e32 v8.l, v10.l
-; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, s0
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s2
-; GFX1200-NEXT: v_lshrrev_b16 v9.h, v12.l, v11.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v7.l
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v8.l
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v11.l, v5.l
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
-; GFX1200-NEXT: v_and_b16 v8.l, v7.h, 1
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.l
-; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s0
-; GFX1200-NEXT: v_and_b16 v7.l, v9.h, v9.l
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v3.l
-; GFX1200-NEXT: v_or_b16 v8.l, v10.l, v8.l
-; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, s0
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s0
+; GFX1200-NEXT: v_and_b16 v5.l, v5.h, v8.h
+; GFX1200-NEXT: v_and_b16 v5.h, v9.h, v9.l
+; GFX1200-NEXT: v_and_b16 v8.h, v6.l, 1
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, v5.l, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h
+; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v8.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v9.l
+; GFX1200-NEXT: v_and_b16 v8.h, v8.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v10.h, v5.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b16 v5.h, v6.h, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v11.h, v9.h
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
+; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, v5.h
; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, s2
-; GFX1200-NEXT: v_or_b16 v9.l, v4.l, v9.l
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v8.l
-; GFX1200-NEXT: v_or_b16 v9.h, v4.l, v9.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v2
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.h, v7.l
-; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
-; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v5.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.l, v1.l
-; GFX1200-NEXT: v_sub_nc_u16 v7.h, v6.h, 1 clamp
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, 8, vcc_lo
+; GFX1200-NEXT: v_or_b16 v8.h, v9.l, v8.h
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
+; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l
+; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h
+; GFX1200-NEXT: v_or_b16 v8.h, v4.l, v9.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v5.l, v6.l, v5.l
+; GFX1200-NEXT: v_min_u16 v6.l, v7.h, 15
+; GFX1200-NEXT: v_or_b16 v3.h, v8.h, v3.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v6.h, s1
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, v9.l
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v6.l, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.l, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
-; GFX1200-NEXT: v_and_b16 v5.l, 0x3ff, v7.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, v7.h, 1
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, s1
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v1.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v2
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, s1
+; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v5.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v3.l
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, 6
+; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
+; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v4.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, 8, s0
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.l, v2.l
+; GFX1200-NEXT: v_lshlrev_b16 v2.l, 3, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0
+; GFX1200-NEXT: v_and_b16 v7.h, v8.l, v7.h
+; GFX1200-NEXT: v_or_b16 v2.h, v8.h, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v2.l, v8.h, v2.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v6.l, v8.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h
+; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v3.l
+; GFX1200-NEXT: v_and_b16 v3.l, v5.l, 63
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v5.h
+; GFX1200-NEXT: v_and_b16 v5.h, v9.l, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v7.l
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 6, v5.l
+; GFX1200-NEXT: v_or_b16 v3.l, v10.l, v5.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v8.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v2.h, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, 7, v10.l
-; GFX1200-NEXT: v_or_b16 v10.l, 0x400, v5.l
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s1
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v9.l
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.l, v2.l
-; GFX1200-NEXT: v_or_b16 v8.h, v9.h, v8.h
-; GFX1200-NEXT: v_and_b16 v4.l, v10.l, v8.l
-; GFX1200-NEXT: v_and_b16 v8.l, v7.l, 63
-; GFX1200-NEXT: v_add_nc_u16 v2.l, v5.h, 6
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v10.l
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
-; GFX1200-NEXT: v_lshlrev_b16 v4.l, 3, v2.l
-; GFX1200-NEXT: v_or_b16 v3.l, v8.h, v3.l
-; GFX1200-NEXT: v_and_b16 v8.h, v5.h, 1
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v2.h, v4.l, 1
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
+; GFX1200-NEXT: v_and_b16 v3.l, v5.h, v3.l
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v13.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v12.l
+; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v5.l
-; GFX1200-NEXT: v_or_b16 v4.l, v9.h, v4.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, v7.h, v10.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v6.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v4.l, v4.l, v4.h
-; GFX1200-NEXT: v_and_b16 v4.h, v5.l, 1
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v2.l
-; GFX1200-NEXT: v_or_b16 v8.l, v9.l, v8.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v7.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, vcc_lo
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
-; GFX1200-NEXT: v_and_b16 v2.l, v7.h, v8.l
-; GFX1200-NEXT: v_or_b16 v4.h, v9.l, v4.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.l, v1.h
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.h
-; GFX1200-NEXT: v_and_b16 v3.h, 0xff, v3.h
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, v2.l, s1
-; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s0
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v9.h, vcc_lo
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.h, v2.h
-; GFX1200-NEXT: v_add_nc_u16 v2.l, v5.h, v2.l
-; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v4.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v5.l, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v8.h, vcc_lo
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v13.l, v13.l
+; GFX1200-NEXT: v_and_b16 v2.h, v5.l, v2.h
+; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v9.l, v3.l
+; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v4.l, v2.h
+; GFX1200-NEXT: v_min_u16 v4.l, v5.l, 15
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.l
; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, s1
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.l, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.l
-; GFX1200-NEXT: v_min_u16 v2.h, v5.h, 15
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v1.h
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo
; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v2.h, 1 clamp
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s1
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v5.h
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
-; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0
; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, v8.l
; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_or_b16 v2.l, v7.l, v2.l
; GFX1200-NEXT: v_add_nc_u16 v8.h, v8.h, -1
-; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 63
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v2.h, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
+; GFX1200-NEXT: v_or_b16 v3.l, v7.l, v3.l
+; GFX1200-NEXT: v_and_b16 v7.l, v6.l, 63
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
; GFX1200-NEXT: v_and_b16 v8.h, v8.l, v8.h
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v4.l, v8.l
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v6.l
-; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v4.h
; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v8.l
+; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1
; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v0.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v2.h
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
-; GFX1200-NEXT: v_or_b16 v4.h, v7.l, v4.h
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
+; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
+; GFX1200-NEXT: v_and_b16 v3.h, 0xff, v3.h
; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v8.h
-; GFX1200-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
+; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v8.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v11, 16, v0
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v4.h, v2.l, vcc_lo
-; GFX1200-NEXT: v_and_b16 v6.l, v6.h, v9.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v3.l, vcc_lo
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.l
+; GFX1200-NEXT: v_and_b16 v4.h, v6.h, v9.h
; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v8.l
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v7.h, v5.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v6.l, s0
-; GFX1200-NEXT: v_min_u16 v4.h, v6.h, 15
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v2.h
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v6.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v9.l, v5.h
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v4.h, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v2.h, v5.l, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.h
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, v6.h, 1
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v7.l
-; GFX1200-NEXT: v_or_b16 v5.l, 0x400, v7.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v7.l, v8.h, -1
-; GFX1200-NEXT: v_add_nc_u16 v1.l, v4.l, 6
+; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s0
+; GFX1200-NEXT: v_min_u16 v3.l, v6.h, 15
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.l
+; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.l, v4.h
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v3.l, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v5.h, 0, s0
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v2.h, v6.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, v7.l
+; GFX1200-NEXT: v_or_b16 v5.h, 0x400, v7.h
; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v9.h, v4.h, v5.l
-; GFX1200-NEXT: v_and_b16 v5.h, v5.l, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v1.l
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v6.h, v5.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
-; GFX1200-NEXT: v_or_b16 v5.h, v8.h, v7.l
-; GFX1200-NEXT: v_or_b16 v7.l, v8.h, v9.l
-; GFX1200-NEXT: v_and_b16 v9.l, v9.h, 1
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, -1
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, 6
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.h, v11.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v3.l, v5.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v1.l, v5.h, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.h, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v1.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v6.h, v5.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v4.l, 0, s0
+; GFX1200-NEXT: v_or_b16 v4.h, v4.h, v2.h
+; GFX1200-NEXT: v_or_b16 v4.l, v8.h, v7.l
+; GFX1200-NEXT: v_and_b16 v7.l, v9.l, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h
+; GFX1200-NEXT: v_or_b16 v2.h, v3.h, v2.l
+; GFX1200-NEXT: v_or_b16 v1.l, v4.l, v1.l
+; GFX1200-NEXT: v_sub_nc_u16 v4.l, 2, v9.h
+; GFX1200-NEXT: v_or_b16 v1.h, v10.l, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v5.l, s1
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v7.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v4.h, vcc_lo
+; GFX1200-NEXT: v_min_u16 v3.h, v4.l, 15
+; GFX1200-NEXT: v_and_b16 v4.l, v6.l, 63
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v11.l
+; GFX1200-NEXT: v_and_b16 v1.h, v5.h, v1.h
+; GFX1200-NEXT: v_sub_nc_u16 v4.h, v3.h, 1 clamp
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.l
+; GFX1200-NEXT: v_and_b16 v6.h, v5.l, 1
+; GFX1200-NEXT: v_and_b16 v4.l, 0x3ff, v3.l
+; GFX1200-NEXT: v_and_b16 v7.l, v3.l, 63
+; GFX1200-NEXT: v_lshlrev_b16 v5.h, v4.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v10.h, v0.h
-; GFX1200-NEXT: v_or_b16 v2.h, v7.l, v2.h
-; GFX1200-NEXT: v_or_b16 v5.h, v5.h, v4.l
-; GFX1200-NEXT: v_or_b16 v4.l, v3.h, v3.l
-; GFX1200-NEXT: v_or_b16 v1.l, v10.l, v9.l
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, 2, v10.h
-; GFX1200-NEXT: v_and_b16 v3.h, v6.l, 63
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.h, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
-; GFX1200-NEXT: v_and_b16 v1.l, v5.l, v1.l
-; GFX1200-NEXT: v_min_u16 v3.l, v7.l, 15
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v0.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v7.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
-; GFX1200-NEXT: v_sub_nc_u16 v5.l, v3.l, 1 clamp
-; GFX1200-NEXT: v_and_b16 v3.h, 0x3ff, v4.h
-; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 63
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 1
-; GFX1200-NEXT: v_lshlrev_b16 v6.h, v5.l, 1
-; GFX1200-NEXT: v_or_b16 v10.l, 0x400, v3.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, 7, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, -1
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
+; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v4.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l
+; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v6.h
+; GFX1200-NEXT: v_lshrrev_b16 v10.h, v3.h, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_or_b16 v7.l, v9.l, v7.l
-; GFX1200-NEXT: v_and_b16 v7.h, v3.h, 1
-; GFX1200-NEXT: v_and_b16 v6.h, v10.l, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, v5.l, v10.l
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v11.l, v3.l, v10.l
-; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v7.h
-; GFX1200-NEXT: v_and_b16 v7.l, v11.l, 1
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v5.h
+; GFX1200-NEXT: v_and_b16 v7.l, v4.l, 1
+; GFX1200-NEXT: v_lshrrev_b16 v3.l, 6, v3.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
+; GFX1200-NEXT: v_and_b16 v5.h, v6.l, v6.h
+; GFX1200-NEXT: v_and_b16 v6.h, v10.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v7.h
+; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.l
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v6.h
-; GFX1200-NEXT: v_add_nc_u16 v1.l, v9.h, v1.l
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v12.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.h
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v4.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v9.l, v1.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v3.h
+; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v6.l
+; GFX1200-NEXT: v_mov_b16_e32 v7.l, v13.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.h
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v4.l, v3.l
+; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v6.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.l, v9.l, v7.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v0
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v8.h, s0
-; GFX1200-NEXT: v_and_b16 v4.h, v5.l, v6.l
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, s3
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v0.l
-; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s2
-; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s1
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.h, v6.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 7, v9.l
-; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v11.l, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s3
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.h, v1.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v3.l
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v6.l
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s2
-; GFX1200-NEXT: v_or_b16 v6.h, v5.l, v6.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s1
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
-; GFX1200-NEXT: v_or_b16 v9.l, v8.l, v9.l
-; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.l
-; GFX1200-NEXT: v_or_b16 v7.h, v8.l, v7.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v6.l
-; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7f, v2.h, s0
-; GFX1200-NEXT: v_or_b16 v1.l, v6.h, v1.l
-; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v4.h
-; GFX1200-NEXT: v_or_b16 v5.h, v7.l, v5.h
-; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s2
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.l
+; GFX1200-NEXT: v_and_b16 v4.l, v4.h, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v8.l, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, 8, s2
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v8.h, s0
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v4.l, s1
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v0.l
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v10.h, v3.h
+; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v0
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
+; GFX1200-NEXT: v_or_b16 v0.h, v4.l, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v6.l, 6
+; GFX1200-NEXT: v_lshlrev_b16 v6.l, 3, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.h, v3.l, s1
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v0.h
-; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 8, s1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v5.h
+; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.l
+; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v6.h
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h
+; GFX1200-NEXT: v_or_b16 v7.h, v7.l, v7.h
+; GFX1200-NEXT: v_or_b16 v1.h, v6.l, v5.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s0
+; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h
+; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1200-NEXT: v_or_b16 v3.l, v7.h, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v5.h, v1.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
-; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v4.l
-; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v0.h
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v11.l, v11.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.l, v3.h, s1
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v11.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
+; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v8.l, s1
-; GFX1200-NEXT: v_or_b32_e32 v6, 0x7f7f0000, v3
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v7.l, s1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v0.h, vcc_lo
+; GFX1200-NEXT: v_or_b32_e32 v5, 0x7f7f0000, v3
+; GFX1200-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0
-; GFX1200-NEXT: v_lshrrev_b64 v[7:8], 24, v[5:6]
-; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v6
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v5
+; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: s_setpc_b64 s[30:31]
@@ -2255,505 +2257,493 @@ define <7 x i8> @to_fp8_v7f16(<7 x half> %x) {
; GFX1200-NEXT: s_wait_kmcnt 0x0
; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.l, v3.l
; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.h, v3.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v3.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v3.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v14, 16, v3
; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v3.l
-; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v4.l
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v4.h
-; GFX1200-NEXT: v_and_b16 v8.l, v4.h, 63
-; GFX1200-NEXT: v_and_b16 v10.l, v5.h, 63
-; GFX1200-NEXT: v_sub_nc_u16 v9.l, 2, v6.h
-; GFX1200-NEXT: v_min_u16 v6.l, v6.l, 15
-; GFX1200-NEXT: v_or_b16 v11.l, 0x400, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v2.l
+; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v4.l
+; GFX1200-NEXT: v_and_b16 v7.l, 0x3ff, v4.h
+; GFX1200-NEXT: v_and_b16 v7.h, v4.h, 63
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.l, v14.l
; GFX1200-NEXT: v_lshrrev_b16 v4.h, 6, v4.h
-; GFX1200-NEXT: v_sub_nc_u16 v9.h, v6.l, 1 clamp
-; GFX1200-NEXT: v_min_u16 v9.l, v9.l, 15
-; GFX1200-NEXT: v_and_b16 v12.l, v7.h, 1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.l
-; GFX1200-NEXT: v_lshlrev_b16 v11.h, v9.h, 1
-; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v5.h
-; GFX1200-NEXT: v_lshrrev_b16 v9.h, v9.h, v11.l
-; GFX1200-NEXT: v_or_b16 v10.l, v13.l, v12.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v11.h, v11.h, -1
-; GFX1200-NEXT: v_lshrrev_b16 v12.l, v6.l, v11.l
-; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v8.h
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v10.l
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, 7, v8.h
-; GFX1200-NEXT: v_and_b16 v11.h, v11.l, v11.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v14.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, v9.l, v8.l
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v11.h
-; GFX1200-NEXT: v_sub_nc_u16 v11.h, v9.l, 1 clamp
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.l, v2.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l
-; GFX1200-NEXT: v_and_b16 v6.l, v12.l, 1
-; GFX1200-NEXT: v_lshlrev_b16 v12.h, v11.h, 1
-; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_min_u16 v5.h, v5.h, 15
+; GFX1200-NEXT: v_or_b16 v10.l, 0x400, v7.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, 7, v7.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.h
+; GFX1200-NEXT: v_sub_nc_u16 v11.l, 2, v9.l
+; GFX1200-NEXT: v_sub_nc_u16 v8.h, v5.h, 1 clamp
+; GFX1200-NEXT: v_lshrrev_b16 v12.h, v5.h, v10.l
+; GFX1200-NEXT: v_and_b16 v12.l, v7.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_sub_nc_u16 v10.h, 2, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
-; GFX1200-NEXT: v_cmp_o_f16_e64 s2, v1.l, v1.l
-; GFX1200-NEXT: v_or_b16 v6.l, v10.l, v6.l
-; GFX1200-NEXT: v_add_nc_u16 v10.l, v12.h, -1
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_and_b16 v6.l, v9.h, v6.l
-; GFX1200-NEXT: v_and_b16 v9.l, v8.l, v10.l
-; GFX1200-NEXT: v_and_b16 v9.h, v8.h, 1
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
-; GFX1200-NEXT: v_lshrrev_b16 v8.l, v11.h, v8.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v6.l, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l
-; GFX1200-NEXT: v_or_b16 v9.l, v11.l, v9.h
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v10.l
-; GFX1200-NEXT: v_and_b16 v9.h, v7.h, 1
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v12.l, v6.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v9.l
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, 6
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v6.l
+; GFX1200-NEXT: v_min_u16 v11.l, v11.l, 15
+; GFX1200-NEXT: v_lshlrev_b16 v10.h, v8.h, 1
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v14.l
+; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l
+; GFX1200-NEXT: v_or_b16 v12.l, v13.l, v12.l
+; GFX1200-NEXT: v_sub_nc_u16 v9.h, 2, v6.h
+; GFX1200-NEXT: v_add_nc_u16 v10.h, v10.h, -1
+; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v8.l
+; GFX1200-NEXT: v_and_b16 v11.h, v8.l, 63
+; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v12.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, 6, v8.l
+; GFX1200-NEXT: v_and_b16 v10.h, v10.l, v10.h
+; GFX1200-NEXT: v_min_u16 v9.h, v9.h, 15
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v11.h
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.l, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v8.h, v10.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v10.h
+; GFX1200-NEXT: v_and_b16 v10.h, v12.h, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v16, 0, 1, s2
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v2.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
+; GFX1200-NEXT: v_sub_nc_u16 v5.h, v11.l, 1 clamp
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, s1
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s1
+; GFX1200-NEXT: v_mov_b16_e32 v12.l, v13.l
+; GFX1200-NEXT: v_or_b16 v13.l, 0x400, v7.h
+; GFX1200-NEXT: v_lshlrev_b16 v13.h, v5.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v3.l
+; GFX1200-NEXT: v_or_b16 v8.h, v12.l, v10.h
+; GFX1200-NEXT: v_mov_b16_e32 v12.l, v15.l
+; GFX1200-NEXT: v_add_nc_u16 v10.l, v13.h, -1
+; GFX1200-NEXT: v_and_b16 v10.h, v7.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v5.h, v13.l
+; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v8.h
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, v12.l
+; GFX1200-NEXT: v_and_b16 v10.l, v13.l, v10.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, v11.l, v13.l
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.h, v5.h
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, s0
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, 6
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v4.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, vcc_lo
-; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.h
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, 3, v4.l
-; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
-; GFX1200-NEXT: v_or_b16 v8.h, v5.l, v8.h
-; GFX1200-NEXT: v_and_b16 v8.l, v8.l, v9.l
-; GFX1200-NEXT: v_or_b16 v9.l, v5.l, v9.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v10.l
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v16.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v11.l
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v12.h, v7.l
+; GFX1200-NEXT: v_and_b16 v11.l, v8.h, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s2
+; GFX1200-NEXT: v_lshlrev_b16 v11.h, 3, v4.l
+; GFX1200-NEXT: v_or_b16 v10.l, v10.l, v10.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v7.l
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v4.l
+; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.l
+; GFX1200-NEXT: v_or_b16 v11.h, v5.l, v11.h
+; GFX1200-NEXT: v_and_b16 v8.l, v8.l, v10.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v10.h, 0, 8, s2
+; GFX1200-NEXT: v_cndmask_b16 v7.l, v7.l, 0, s2
+; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v11.l
+; GFX1200-NEXT: v_or_b16 v4.l, v11.h, v4.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; GFX1200-NEXT: v_or_b16 v10.h, v5.l, v10.h
+; GFX1200-NEXT: v_cmp_o_f16_e64 s2, v1.l, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v5.h, s0
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_or_b16 v4.h, v10.h, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.h, v8.l
+; GFX1200-NEXT: v_sub_nc_u16 v8.l, v9.h, 1 clamp
+; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v4.h, s1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v8.h, v5.h
+; GFX1200-NEXT: v_lshlrev_b16 v5.h, v8.l, 1
+; GFX1200-NEXT: v_or_b16 v8.h, 0x400, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, v8.l, s1
-; GFX1200-NEXT: v_or_b16 v4.l, v9.l, v4.h
-; GFX1200-NEXT: v_or_b16 v4.h, v8.h, v6.l
-; GFX1200-NEXT: v_min_u16 v6.l, v10.h, 15
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.h, v2.l
-; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, v8.l
-; GFX1200-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v4.h, s0
-; GFX1200-NEXT: v_sub_nc_u16 v4.h, v6.l, 1 clamp
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v4.h
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, -1
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v3.l, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
+; GFX1200-NEXT: v_mov_b16_e32 v5.l, v10.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v3
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v7.h
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, v8.l
-; GFX1200-NEXT: v_and_b16 v8.l, 0x3ff, v8.h
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, v4.h, 1
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, 7, v10.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, s0
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6
-; GFX1200-NEXT: v_or_b16 v10.h, 0x400, v8.l
-; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
-; GFX1200-NEXT: v_and_b16 v11.h, v8.h, 63
-; GFX1200-NEXT: v_cndmask_b16 v7.h, v7.h, 0, s0
-; GFX1200-NEXT: v_lshlrev_b16 v11.l, 3, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v12.l, v6.l, v10.h
-; GFX1200-NEXT: v_and_b16 v9.l, v10.h, v9.l
-; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.l, 7, v8.l
-; GFX1200-NEXT: v_or_b16 v11.l, v10.l, v11.l
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, v4.h, v10.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.l
-; GFX1200-NEXT: v_and_b16 v9.l, v12.l, 1
-; GFX1200-NEXT: v_or_b16 v7.h, v9.h, v7.h
-; GFX1200-NEXT: v_and_b16 v9.h, v8.l, 1
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v3.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v11.h
-; GFX1200-NEXT: v_or_b16 v5.h, v11.l, v5.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, 6, v8.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
-; GFX1200-NEXT: v_or_b16 v9.l, v13.l, v9.l
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v4.h, 0, s1
+; GFX1200-NEXT: v_and_b16 v4.h, v8.h, v5.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, s1
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v3.l, v3.l
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v9.l
-; GFX1200-NEXT: v_or_b16 v6.h, v14.l, v9.h
+; GFX1200-NEXT: v_cndmask_b16 v4.l, 0x7f, v4.l, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v9.l, v5.l
+; GFX1200-NEXT: v_cndmask_b16 v5.l, 0, 8, s1
+; GFX1200-NEXT: v_lshlrev_b16 v5.h, 7, v10.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v9.h, v8.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 6
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v14.l, v14.l
+; GFX1200-NEXT: v_or_b16 v4.h, v5.h, v5.l
+; GFX1200-NEXT: v_and_b16 v10.l, v9.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.h, v7.h, vcc_lo
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v3.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.h, v2.h
-; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v4.h, s0
-; GFX1200-NEXT: v_and_b16 v5.h, v8.h, v6.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.h, v2.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX1200-NEXT: v_lshlrev_b16 v5.l, 3, v3.l
+; GFX1200-NEXT: v_and_b16 v4.l, 0xff, v4.l
+; GFX1200-NEXT: v_or_b16 v3.h, v4.h, v3.h
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v7.l, 0, s0
+; GFX1200-NEXT: v_and_b16 v7.l, v6.l, 63
+; GFX1200-NEXT: v_or_b16 v5.l, v5.h, v5.l
+; GFX1200-NEXT: v_or_b16 v10.l, v11.l, v10.l
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v8.l, v8.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v8.l, v12.l
+; GFX1200-NEXT: v_or_b16 v4.h, v5.l, v4.h
+; GFX1200-NEXT: v_and_b16 v5.l, v7.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, v10.l, vcc_lo
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v3.h, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v12.l, v3.l
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.l, v5.h
-; GFX1200-NEXT: v_sub_nc_u16 v3.h, 2, v4.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v10.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.h
+; GFX1200-NEXT: v_sub_nc_u16 v3.l, 2, v8.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, 0x7f, v4.l, s1
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v4.h, v3.h, s0
+; GFX1200-NEXT: v_or_b16 v5.l, v11.l, v5.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v2.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.l, 0x7f, v5.l, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v3.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v5.h
-; GFX1200-NEXT: v_min_u16 v3.h, v3.h, 15
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.l, v2.l
-; GFX1200-NEXT: v_and_b16 v4.l, 0xff, v4.l
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, vcc_lo
+; GFX1200-NEXT: v_min_u16 v3.l, v3.l, 15
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v14.l
+; GFX1200-NEXT: v_and_b16 v4.h, v6.l, v5.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v12.l
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v9.l, v7.l
+; GFX1200-NEXT: v_sub_nc_u16 v7.l, v3.l, 1 clamp
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v5.h, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.h, v4.h
+; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v6.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, v7.l, 1
+; GFX1200-NEXT: v_and_b16 v8.h, 0x80, v8.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
+; GFX1200-NEXT: v_or_b16 v9.l, 0x400, v5.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 8, v2.l
-; GFX1200-NEXT: v_sub_nc_u16 v8.l, v3.h, 1 clamp
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s0
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v7.l, v6.l
-; GFX1200-NEXT: v_and_b16 v6.h, 0x80, v6.h
-; GFX1200-NEXT: v_and_b16 v7.l, 0x3ff, v7.h
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.l, 1
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 6
-; GFX1200-NEXT: v_or_b16 v8.h, v6.h, v8.h
-; GFX1200-NEXT: v_or_b16 v9.h, 0x400, v7.l
-; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, 7, v7.l
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, 3, v6.l
-; GFX1200-NEXT: v_or_b16 v3.l, v8.h, v3.l
-; GFX1200-NEXT: v_lshrrev_b16 v10.h, v3.h, v9.h
-; GFX1200-NEXT: v_and_b16 v8.h, v9.h, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v8.l, v8.l, v9.h
-; GFX1200-NEXT: v_or_b16 v9.l, v6.h, v10.l
-; GFX1200-NEXT: v_and_b16 v10.l, v7.h, 63
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 6, v7.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
-; GFX1200-NEXT: v_and_b16 v8.h, v10.h, 1
-; GFX1200-NEXT: v_or_b16 v5.h, v9.l, v5.h
-; GFX1200-NEXT: v_and_b16 v9.l, v7.l, 1
-; GFX1200-NEXT: v_lshlrev_b16 v5.l, 8, v5.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h
+; GFX1200-NEXT: v_or_b16 v9.h, v8.h, v9.h
+; GFX1200-NEXT: v_and_b16 v7.h, v9.l, v7.h
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v9.l
+; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, v10.l
+; GFX1200-NEXT: v_lshrrev_b16 v10.l, v3.l, v9.l
+; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v5.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h
+; GFX1200-NEXT: v_and_b16 v7.h, v6.l, 63
+; GFX1200-NEXT: v_add_nc_u16 v6.h, v6.h, 6
+; GFX1200-NEXT: v_and_b16 v9.h, v10.l, 1
+; GFX1200-NEXT: v_and_b16 v9.l, v5.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v10.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v11.h, v1.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v11.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.l
-; GFX1200-NEXT: v_or_b16 v8.h, v10.l, v8.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v6.h
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
+; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v5.h, v3.l, vcc_lo
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
-; GFX1200-NEXT: v_and_b16 v8.l, v8.l, v8.h
-; GFX1200-NEXT: v_or_b16 v6.l, v11.l, v9.l
-; GFX1200-NEXT: v_sub_nc_u16 v9.l, 2, v11.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v7.h, v8.h, v7.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v3.h, s1
+; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v9.h
+; GFX1200-NEXT: v_mov_b16_e32 v11.l, v13.l
+; GFX1200-NEXT: v_or_b16 v4.h, v7.h, v4.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX1200-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v6.h, vcc_lo
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_and_b16 v5.h, v7.h, v6.l
-; GFX1200-NEXT: v_min_u16 v3.h, v9.l, 15
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v8.l, s0
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.h, v1.l
-; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, s1
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v7.l, v5.h
-; GFX1200-NEXT: v_sub_nc_u16 v8.l, v3.h, 1 clamp
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.h, v6.l
-; GFX1200-NEXT: v_and_b16 v6.h, 0x3ff, v7.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v2
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, v8.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, vcc_lo
+; GFX1200-NEXT: v_or_b16 v3.l, v11.l, v9.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.l, v1.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.h
+; GFX1200-NEXT: v_lshrrev_b32_e32 v11, 31, v2
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v3.l, v6.l, v3.l
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, 2, v9.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, v5.l, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.l, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v9.h, 7, v11.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v5.h, v3.l
+; GFX1200-NEXT: v_min_u16 v5.l, v6.h, 15
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v1.l
; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v6.l
-; GFX1200-NEXT: v_or_b16 v8.h, 0x400, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v6.h
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.l
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v5.l, 1 clamp
+; GFX1200-NEXT: v_and_b16 v2.h, 0x3ff, v5.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s0
+; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s0
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, v6.h, 1
+; GFX1200-NEXT: v_or_b16 v10.h, 0x400, v2.h
+; GFX1200-NEXT: v_or_b16 v7.h, v9.h, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, v8.h, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, v10.l
; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, -1
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v6.l, 0, s0
-; GFX1200-NEXT: v_lshrrev_b16 v9.h, v3.h, v8.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.l, v8.l, v8.h
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, v9.l
-; GFX1200-NEXT: v_and_b16 v6.l, v8.h, v7.l
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 8, s0
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, 7, v10.l
-; GFX1200-NEXT: v_and_b16 v10.l, v9.h, 1
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 6
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
-; GFX1200-NEXT: v_or_b16 v6.l, v9.l, v7.l
-; GFX1200-NEXT: v_and_b16 v7.l, v7.h, 63
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, 3, v4.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_and_b16 v8.h, v6.h, 1
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
-; GFX1200-NEXT: v_or_b16 v10.h, v9.l, v10.h
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
-; GFX1200-NEXT: v_or_b16 v2.l, v6.l, v2.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v7.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v7.h, v1.h
-; GFX1200-NEXT: v_or_b16 v7.l, v11.l, v10.l
-; GFX1200-NEXT: v_or_b16 v5.h, v10.h, v5.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l
-; GFX1200-NEXT: v_and_b16 v7.l, v8.l, v7.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_or_b16 v3.h, v10.l, v8.h
+; GFX1200-NEXT: v_and_b16 v8.h, v5.h, 63
+; GFX1200-NEXT: v_or_b16 v6.l, v7.h, v6.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, v5.l, v10.h
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, 6
+; GFX1200-NEXT: v_and_b16 v7.l, v10.h, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s1
+; GFX1200-NEXT: v_lshrrev_b16 v2.h, 7, v2.h
+; GFX1200-NEXT: v_and_b16 v10.l, v7.h, 1
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v10.h
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 3, v8.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 6, v5.h
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.l, v2.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, v7.l, vcc_lo
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v3.h, v6.l, v3.h
-; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v7.h
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.h, v7.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v5.h, v2.l, vcc_lo
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.h
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.h, v3.h
-; GFX1200-NEXT: v_min_u16 v5.h, v6.l, 15
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v1.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v1.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v3.h
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, v5.h, 1 clamp
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
+; GFX1200-NEXT: v_or_b16 v7.l, v9.h, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s0
-; GFX1200-NEXT: v_and_b16 v6.l, 0x80, v6.l
-; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v6.h
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, v7.l, 1
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
-; GFX1200-NEXT: v_or_b16 v8.l, v6.l, v8.l
-; GFX1200-NEXT: v_or_b16 v10.h, 0x400, v8.h
+; GFX1200-NEXT: v_cndmask_b16 v4.h, 0x7f, v4.h, s1
+; GFX1200-NEXT: v_or_b16 v8.h, v11.l, v10.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v9.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v3.l, v7.l, v3.l
+; GFX1200-NEXT: v_and_b16 v7.l, v2.h, 1
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v8.l
+; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v13.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v6.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, v6.h, s0
+; GFX1200-NEXT: v_or_b16 v5.l, v10.l, v7.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v13.l
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v12.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v13.l
+; GFX1200-NEXT: v_add_nc_u16 v2.l, v7.h, v2.l
+; GFX1200-NEXT: v_and_b16 v5.l, v5.h, v5.l
+; GFX1200-NEXT: v_sub_nc_u16 v5.h, 2, v6.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v9.h, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, v5.l
+; GFX1200-NEXT: v_min_u16 v5.l, v5.h, 15
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 8, v1.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v2.h
+; GFX1200-NEXT: v_sub_nc_u16 v7.l, v5.l, 1 clamp
+; GFX1200-NEXT: v_and_b16 v5.h, 0x80, v5.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, vcc_lo
+; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v12.l, v12.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0
+; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v6.l
+; GFX1200-NEXT: v_lshlrev_b16 v9.h, v7.l, 1
+; GFX1200-NEXT: v_or_b16 v7.h, v5.h, v7.h
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v9.l, v8.l
+; GFX1200-NEXT: v_or_b16 v9.l, 0x400, v8.h
; GFX1200-NEXT: v_add_nc_u16 v9.h, v9.h, -1
-; GFX1200-NEXT: v_add_nc_u16 v10.l, v11.h, v10.l
-; GFX1200-NEXT: v_or_b16 v4.h, v8.l, v4.h
-; GFX1200-NEXT: v_and_b16 v8.l, v6.h, 63
+; GFX1200-NEXT: v_or_b16 v2.l, v7.h, v2.l
+; GFX1200-NEXT: v_and_b16 v7.h, v6.l, 63
; GFX1200-NEXT: v_lshrrev_b16 v8.h, 7, v8.h
-; GFX1200-NEXT: v_and_b16 v9.h, v10.h, v9.h
-; GFX1200-NEXT: v_add_nc_u16 v9.l, v10.l, 6
-; GFX1200-NEXT: v_lshrrev_b16 v10.l, v5.h, v10.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 3, v9.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v10.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.h
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, 6
+; GFX1200-NEXT: v_and_b16 v9.h, v9.l, v9.h
+; GFX1200-NEXT: v_lshrrev_b16 v10.l, v5.l, v9.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v7.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 6, v6.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v9.l
+; GFX1200-NEXT: v_and_b16 v10.h, v10.l, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.h
; GFX1200-NEXT: v_and_b16 v9.h, v8.h, 1
-; GFX1200-NEXT: v_and_b16 v11.h, v10.l, 1
-; GFX1200-NEXT: v_or_b16 v8.l, v6.l, v8.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v10.h, v0.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v9.l
-; GFX1200-NEXT: v_or_b16 v3.h, v8.l, v3.h
-; GFX1200-NEXT: v_sub_nc_u16 v8.l, 2, v10.h
-; GFX1200-NEXT: v_or_b16 v11.l, v12.l, v11.h
-; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v9.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v5.h
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v2.h, v2.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v3.h, v4.h, vcc_lo
-; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v11.l
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.h, v6.h
-; GFX1200-NEXT: v_min_u16 v3.h, v8.l, 15
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v0.l
+; GFX1200-NEXT: v_or_b16 v7.h, v5.h, v7.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v9.l, v0.l
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v8.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s1
-; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v7.l, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, v3.h, 1 clamp
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s1
+; GFX1200-NEXT: v_or_b16 v9.h, v11.l, v9.h
+; GFX1200-NEXT: v_or_b16 v2.h, v7.h, v2.h
+; GFX1200-NEXT: v_sub_nc_u16 v7.h, 2, v9.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l
+; GFX1200-NEXT: v_or_b16 v10.h, v12.l, v10.h
+; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v9.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.h, v2.l, s0
+; GFX1200-NEXT: v_min_u16 v2.h, v7.h, 15
; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v10.l, v4.h
+; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v10.h
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v8.h, v6.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
-; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v6.h
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, v7.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7f, v3.l, vcc_lo
+; GFX1200-NEXT: v_sub_nc_u16 v7.h, v2.h, 1 clamp
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v6.l, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v7.h, v8.l
-; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v8.h
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v9.l, -1
-; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v1
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v5.h, s0
+; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v7.l, s1
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v5.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.l, v0.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7f, v2.l, s2
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.l, v6.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
-; GFX1200-NEXT: v_add_nc_u16 v1.l, v6.l, 6
-; GFX1200-NEXT: v_lshrrev_b16 v10.l, v3.h, v7.h
-; GFX1200-NEXT: v_and_b16 v6.l, v7.h, v8.l
-; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s0
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, 7, v9.l
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, 3, v1.l
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s1
+; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v7.l
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, v7.h, 1
+; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v1
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 7, v6.l
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v6.h, v8.l
+; GFX1200-NEXT: v_or_b16 v6.h, 0x400, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.h, -1
+; GFX1200-NEXT: v_lshlrev_b16 v9.h, 7, v10.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 8, s0
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, 6
+; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s0
+; GFX1200-NEXT: v_and_b16 v1.l, v6.h, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v10.l, v2.h, v6.h
+; GFX1200-NEXT: v_or_b16 v8.l, v9.h, v8.h
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, 3, v1.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v10.h, v14.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v5.l, 0, s1
+; GFX1200-NEXT: v_or_b16 v6.l, v8.l, v6.l
+; GFX1200-NEXT: v_or_b16 v5.l, v9.h, v8.h
+; GFX1200-NEXT: v_and_b16 v8.h, v10.l, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v1.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v7.h, v6.h
+; GFX1200-NEXT: v_or_b16 v1.l, v5.l, v1.l
+; GFX1200-NEXT: v_sub_nc_u16 v5.l, 2, v10.h
+; GFX1200-NEXT: v_or_b16 v1.h, v11.l, v8.h
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v2.h, v14.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.l, s0
+; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
+; GFX1200-NEXT: v_and_b16 v6.l, v7.l, 63
+; GFX1200-NEXT: v_and_b16 v1.h, v6.h, v1.h
+; GFX1200-NEXT: v_and_b16 v8.h, v2.h, 63
+; GFX1200-NEXT: v_and_b16 v8.l, v5.h, 1
+; GFX1200-NEXT: v_sub_nc_u16 v6.h, v5.l, 1 clamp
; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.l
-; GFX1200-NEXT: v_or_b16 v6.l, v9.l, v8.l
-; GFX1200-NEXT: v_or_b16 v8.l, v9.l, v9.h
-; GFX1200-NEXT: v_and_b16 v9.h, v10.l, 1
+; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v2.h
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, 6, v7.l
+; GFX1200-NEXT: v_lshrrev_b16 v2.h, 6, v2.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, v6.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v11.h, v0.h
-; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v4.h
-; GFX1200-NEXT: v_or_b16 v5.h, v8.l, v5.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v7.h
-; GFX1200-NEXT: v_or_b16 v1.l, v11.l, v9.h
-; GFX1200-NEXT: v_sub_nc_u16 v6.l, 2, v11.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v8.h
+; GFX1200-NEXT: v_or_b16 v11.h, 0x400, v6.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l
+; GFX1200-NEXT: v_add_nc_u16 v7.h, v7.h, -1
+; GFX1200-NEXT: v_or_b16 v8.l, v11.l, v8.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v5.h, v4.h, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v0.h
-; GFX1200-NEXT: v_min_u16 v5.h, v6.l, 15
-; GFX1200-NEXT: v_and_b16 v6.l, v6.h, 63
-; GFX1200-NEXT: v_and_b16 v1.l, v7.l, v1.l
-; GFX1200-NEXT: v_and_b16 v8.h, v7.h, 1
-; GFX1200-NEXT: v_and_b16 v9.h, v3.h, 63
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, v5.h, 1 clamp
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.l
-; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v3.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 6, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, 6, v3.h
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, v7.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, v1.l, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.h
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
+; GFX1200-NEXT: v_and_b16 v8.h, v6.l, 1
+; GFX1200-NEXT: v_and_b16 v7.h, v11.h, v7.h
+; GFX1200-NEXT: v_and_b16 v7.l, v7.l, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v11.h
; GFX1200-NEXT: v_mov_b16_e32 v11.l, v12.l
-; GFX1200-NEXT: v_or_b16 v12.l, 0x400, v6.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_add_nc_u16 v1.l, v10.l, v1.l
-; GFX1200-NEXT: v_or_b16 v8.h, v11.l, v8.h
-; GFX1200-NEXT: v_and_b16 v8.l, v12.l, v8.l
-; GFX1200-NEXT: v_and_b16 v9.h, v6.l, 1
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v13.l
-; GFX1200-NEXT: v_lshrrev_b16 v12.h, v5.h, v12.l
-; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.l, v12.l
-; GFX1200-NEXT: v_or_b16 v8.l, v11.l, v9.h
-; GFX1200-NEXT: v_and_b16 v8.h, v12.h, 1
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v7.h, v6.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7f, v2.h, s2
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v8.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.l
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v6.h
-; GFX1200-NEXT: v_mov_b16_e32 v11.l, v13.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.h
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.l, v3.h
+; GFX1200-NEXT: v_lshrrev_b16 v12.l, v5.l, v11.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.h
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v7.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l
+; GFX1200-NEXT: v_or_b16 v7.h, v11.l, v8.h
+; GFX1200-NEXT: v_and_b16 v8.l, v12.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s2
+; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v10.l, v1.h
+; GFX1200-NEXT: v_and_b16 v2.h, v2.h, v7.h
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v13.l
+; GFX1200-NEXT: v_mov_b16_e32 v11.l, v15.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v6.h, v6.h, 0, vcc_lo
-; GFX1200-NEXT: v_or_b16 v7.h, v11.l, v8.h
+; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 7, v1.h
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v6.l, v2.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v9.h, s0
+; GFX1200-NEXT: v_or_b16 v7.l, v11.l, v8.l
; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v3.h
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, v9.l, s0
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v1.h, v1.h
-; GFX1200-NEXT: v_and_b16 v6.l, v7.l, v7.h
-; GFX1200-NEXT: v_mov_b16_e32 v7.l, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s2
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 7, v2.h
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v13.l, v13.l
+; GFX1200-NEXT: v_and_b16 v6.l, v6.h, v7.l
+; GFX1200-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.h, 8, v0.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s3
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3
; GFX1200-NEXT: v_cndmask_b16 v7.h, 0, 8, s2
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s3
-; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v6.l, s1
+; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v6.l, s1
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v9.l, v5.l
+; GFX1200-NEXT: v_and_b16 v6.h, 0x80, v6.h
; GFX1200-NEXT: v_add_nc_u16 v7.l, v10.h, v7.l
-; GFX1200-NEXT: v_mov_b16_e32 v8.l, v13.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v10, 31, v0
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v0.l
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v12.h, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v12.l, v6.l
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 6
+; GFX1200-NEXT: v_or_b16 v0.h, v6.h, v7.h
; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, 6
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v11.h, v8.l
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, 7, v10.l
-; GFX1200-NEXT: v_and_b16 v6.l, 0x80, v6.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v5.h
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, 3, v7.l
-; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, 6
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v7.l
-; GFX1200-NEXT: v_or_b16 v7.h, v6.l, v7.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v9.h, 0, 8, s1
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v5.h, 0, s1
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, 3, v8.l
-; GFX1200-NEXT: v_or_b16 v8.h, v6.l, v8.h
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v8.l
-; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h
-; GFX1200-NEXT: v_or_b16 v1.l, v7.h, v1.l
-; GFX1200-NEXT: v_or_b16 v10.h, v10.l, v10.h
-; GFX1200-NEXT: v_or_b16 v6.h, v8.h, v6.h
-; GFX1200-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX1200-NEXT: v_or_b16 v5.h, v9.h, v5.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v3.h, v10.h, v3.h
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v10.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 7, v6.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, 3, v5.l
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s3
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 3, v7.l
+; GFX1200-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 8, s1
+; GFX1200-NEXT: v_or_b16 v7.h, v6.h, v7.h
+; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s1
+; GFX1200-NEXT: v_or_b16 v9.l, v8.h, v9.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v5.l
+; GFX1200-NEXT: v_or_b16 v8.l, v8.h, v8.l
+; GFX1200-NEXT: v_or_b16 v1.h, v7.h, v5.h
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v7.l
+; GFX1200-NEXT: v_or_b16 v2.h, v9.l, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, s0
+; GFX1200-NEXT: v_or_b16 v5.l, v8.l, v6.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v6.h, v1.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v1.h, v0.h, vcc_lo
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
-; GFX1200-NEXT: v_or_b16 v6.h, v4.l, v5.l
+; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v3.h, v5.h, s1
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7f, v4.h, s0
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v0.h
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v2.h, v5.l, s1
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v14.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v6.h, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
-; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v3.h
+; GFX1200-NEXT: v_lshlrev_b16 v2.h, 8, v3.l
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v10.l, s1
-; GFX1200-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX1200-NEXT: v_lshlrev_b16 v3.l, 8, v2.l
+; GFX1200-NEXT: v_cndmask_b16 v0.l, v1.h, v8.h, s1
+; GFX1200-NEXT: v_and_b16 v1.h, 0xff, v4.h
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v14.l, v14.l
+; GFX1200-NEXT: v_or_b16 v4.h, v4.l, v3.h
+; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7f, v1.l, vcc_lo
-; GFX1200-NEXT: v_or_b16 v2.l, v2.h, v0.h
+; GFX1200-NEXT: v_cndmask_b16 v0.h, 0x7f, v0.h, vcc_lo
+; GFX1200-NEXT: v_or_b16 v4.l, v1.h, v2.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, s0
-; GFX1200-NEXT: v_or_b16 v6.l, v1.h, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v0.h
+; GFX1200-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v4
; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX1200-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX1200-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX1200-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX1200-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-TRUE16-LABEL: to_fp8_v7f16:
@@ -2761,20 +2751,21 @@ define <7 x i8> @to_fp8_v7f16(<7 x half> %x) {
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v4.l, v1
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v7.h, v3
-; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v7.l, v2
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.h, v3
+; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v3.l, v2
; GFX1250-TRUE16-NEXT: v_cvt_pk_fp8_f16_e64 v0.l, v0
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v7
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
-; GFX1250-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[6:7]
+; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[2:3]
; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.h
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.h
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1250-FAKE16-LABEL: to_fp8_v7f16:
@@ -3213,448 +3204,441 @@ define <5 x i8> @to_bf8_v5f16(<5 x half> %x) {
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v2.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.h, v2.l
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v2.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.l, v2.h
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.h, v2.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v3.l, v2.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 31, v2
; GFX1200-NEXT: v_lshrrev_b16 v4.l, 8, v2.l
-; GFX1200-NEXT: v_sub_nc_u16 v4.h, -5, v3.l
-; GFX1200-NEXT: v_and_b16 v6.h, 0x3ff, v3.h
-; GFX1200-NEXT: v_and_b16 v7.l, 0x7f, v3.h
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, 7, v3.h
-; GFX1200-NEXT: v_sub_nc_u16 v8.h, -5, v5.h
+; GFX1200-NEXT: v_sub_nc_u16 v4.h, -5, v3.h
+; GFX1200-NEXT: v_and_b16 v2.h, 0x3ff, v3.l
+; GFX1200-NEXT: v_and_b16 v5.l, 0x7f, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v3.l, 7, v3.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v12.l
; GFX1200-NEXT: v_min_u16 v4.h, v4.h, 15
-; GFX1200-NEXT: v_or_b16 v10.h, 0x400, v6.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 8, v6.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
-; GFX1200-NEXT: v_and_b16 v9.l, 0x7f, v5.l
-; GFX1200-NEXT: v_sub_nc_u16 v8.l, v4.h, 1 clamp
-; GFX1200-NEXT: v_min_u16 v8.h, v8.h, 15
-; GFX1200-NEXT: v_and_b16 v11.h, v6.h, 1
+; GFX1200-NEXT: v_or_b16 v7.h, 0x400, v2.h
+; GFX1200-NEXT: v_lshrrev_b16 v2.h, 8, v2.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v5.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.h, v12.l
+; GFX1200-NEXT: v_sub_nc_u16 v6.l, v4.h, 1 clamp
+; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v5.h
+; GFX1200-NEXT: v_and_b16 v9.l, v2.h, 1
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX1200-NEXT: v_lshrrev_b16 v12.h, v4.h, v10.h
-; GFX1200-NEXT: v_lshlrev_b16 v10.l, v8.l, 1
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v5.l
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v5.l
-; GFX1200-NEXT: v_or_b16 v11.h, v12.l, v11.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
+; GFX1200-NEXT: v_sub_nc_u16 v9.h, -5, v6.h
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, v6.l, 1
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.l, v7.h
+; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
+; GFX1200-NEXT: v_lshrrev_b16 v10.l, v4.h, v7.h
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
+; GFX1200-NEXT: v_or_b16 v4.h, 0x400, v8.h
+; GFX1200-NEXT: v_and_b16 v10.h, 0x7f, v5.h
+; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v9.l
+; GFX1200-NEXT: v_and_b16 v9.l, v10.l, 1
+; GFX1200-NEXT: v_and_b16 v8.l, v7.h, v8.l
; GFX1200-NEXT: v_and_b16 v4.l, 0x80, v4.l
-; GFX1200-NEXT: v_add_nc_u16 v7.l, v10.l, -1
-; GFX1200-NEXT: v_or_b16 v11.l, 0x400, v7.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v7.h
-; GFX1200-NEXT: v_and_b16 v3.h, v3.h, v11.h
-; GFX1200-NEXT: v_sub_nc_u16 v11.h, v8.h, 1 clamp
-; GFX1200-NEXT: v_and_b16 v7.l, v10.h, v7.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v8.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v6.l, v1.l
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v6.h, v3.h
-; GFX1200-NEXT: v_or_b16 v10.l, 0x7c, v4.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v7.l
-; GFX1200-NEXT: v_and_b16 v7.l, v12.h, 1
-; GFX1200-NEXT: v_sub_nc_u16 v9.h, -5, v6.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.h
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v9.l
-; GFX1200-NEXT: v_cmp_class_f16_e64 s5, v2.l, 0x204
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v4.h, v8.l, v10.h
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s1
-; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v7.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, v11.h, 1
-; GFX1200-NEXT: v_mov_b16_e32 v12.l, v14.l
-; GFX1200-NEXT: v_lshrrev_b16 v8.l, v8.h, v11.l
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
-; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v6.h
-; GFX1200-NEXT: v_add_nc_u16 v6.h, v7.l, -1
-; GFX1200-NEXT: v_and_b16 v7.l, v7.h, 1
-; GFX1200-NEXT: v_and_b16 v8.h, v8.l, 1
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v9.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, s0
-; GFX1200-NEXT: v_and_b16 v6.h, v11.l, v6.h
-; GFX1200-NEXT: v_or_b16 v7.l, v12.l, v7.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 14
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v12.h, v4.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s0, 0, v6.h
-; GFX1200-NEXT: v_and_b16 v5.l, v5.l, v7.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v11.h, v11.l
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, 2, v3.l
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s3, 30, v3.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s3, 0, v10.h
+; GFX1200-NEXT: v_add_nc_u16 v2.h, v2.h, v3.l
+; GFX1200-NEXT: v_lshrrev_b16 v3.l, 8, v8.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v8.l
+; GFX1200-NEXT: v_min_u16 v8.l, v9.h, 15
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v7.h, v5.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 30, v3.l
-; GFX1200-NEXT: s_and_b32 s1, s3, s1
-; GFX1200-NEXT: v_mov_b16_e32 v9.l, v13.l
-; GFX1200-NEXT: v_cndmask_b16 v7.l, 0, 4, s0
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, s0
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s0, 1, v3.l
-; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_or_b32 s1, s4, s1
-; GFX1200-NEXT: v_or_b16 v8.h, v9.l, v8.h
-; GFX1200-NEXT: v_or_b16 v9.l, v4.l, v10.h
-; GFX1200-NEXT: v_or_b16 v7.l, v4.l, v7.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v8.h
-; GFX1200-NEXT: v_or_b16 v3.h, v9.l, v3.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_or_b16 v4.h, v7.l, v4.h
-; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v6.h, s2
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.h, v1.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s3
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v2.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s2
+; GFX1200-NEXT: v_sub_nc_u16 v7.h, v8.l, 1 clamp
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v2.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s3, 0, v8.l
+; GFX1200-NEXT: v_cmp_class_f16_e64 s6, v2.l, 0x204
+; GFX1200-NEXT: v_or_b16 v8.h, v11.l, v9.l
+; GFX1200-NEXT: v_lshrrev_b16 v9.l, v8.l, v4.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s2
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s2
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v2.l, v2.l
+; GFX1200-NEXT: v_and_b16 v6.l, v6.l, v8.h
+; GFX1200-NEXT: v_lshlrev_b16 v8.h, v7.h, 1
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, v11.l
+; GFX1200-NEXT: v_mov_b16_e32 v11.l, v14.l
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v2.l, v1.l
+; GFX1200-NEXT: v_cndmask_b16 v6.l, 0, v6.l, s1
+; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.h, -1
+; GFX1200-NEXT: v_and_b16 v8.h, v3.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v3.h, 14
+; GFX1200-NEXT: v_and_b16 v9.h, v9.l, 1
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v10.l, v6.l
+; GFX1200-NEXT: v_and_b16 v8.l, v4.h, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v7.h, v4.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, 2, v3.h
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s2, 1, v3.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v6.l
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s4, 0, v8.l
+; GFX1200-NEXT: v_or_b16 v8.l, v11.l, v8.h
+; GFX1200-NEXT: v_or_b16 v7.h, v4.l, v7.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s5, 30, v3.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 4, s1
+; GFX1200-NEXT: v_cndmask_b16 v6.l, v6.l, 0, s1
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s4
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v2.h
+; GFX1200-NEXT: v_or_b16 v2.h, v7.h, v2.h
+; GFX1200-NEXT: v_or_b16 v8.h, v4.l, v8.h
+; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v8.l
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s4, 30, v3.h
+; GFX1200-NEXT: v_sub_nc_u16 v7.l, -5, v2.l
+; GFX1200-NEXT: v_or_b16 v5.l, 0x7c, v4.l
+; GFX1200-NEXT: v_or_b16 v6.l, v8.h, v6.l
+; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v5.h
+; GFX1200-NEXT: s_and_b32 s1, s4, s1
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.h, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v5.l
-; GFX1200-NEXT: v_min_u16 v4.h, v9.h, 15
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.l, v3.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 31, v2
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v6.l, s2
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v10.l, s1
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
-; GFX1200-NEXT: v_sub_nc_u16 v7.h, v4.h, 1 clamp
-; GFX1200-NEXT: v_and_b16 v9.h, 0x7f, v6.h
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, 7, v9.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.l, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v5.h, v7.l
-; GFX1200-NEXT: v_and_b16 v5.h, 0x3ff, v6.h
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, v7.h, 1
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.l
-; GFX1200-NEXT: v_cndmask_b16 v5.l, v5.l, 0, s0
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v4.l, 14
-; GFX1200-NEXT: v_or_b16 v9.l, 0x400, v5.h
+; GFX1200-NEXT: s_or_b32 s1, s5, s1
+; GFX1200-NEXT: v_and_b16 v3.h, v4.h, v9.h
+; GFX1200-NEXT: v_min_u16 v4.h, v7.l, 15
+; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.l, s1
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.l
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, v3.h, s3
+; GFX1200-NEXT: v_sub_nc_u16 v5.h, v4.h, 1 clamp
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v4.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v4.l, vcc_lo
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s1
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v4.l, v1.l
+; GFX1200-NEXT: v_add_nc_u16 v3.h, v9.l, v3.h
+; GFX1200-NEXT: v_lshlrev_b16 v7.h, 7, v13.l
+; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.l, s6
+; GFX1200-NEXT: v_mov_b16_e32 v6.l, v7.l
+; GFX1200-NEXT: v_and_b16 v5.l, 0x3ff, v4.l
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, v5.h, 1
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.h
+; GFX1200-NEXT: v_and_b16 v8.h, 0x7f, v4.l
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.h, v6.l
+; GFX1200-NEXT: v_or_b16 v8.l, 0x400, v5.l
; GFX1200-NEXT: v_add_nc_u16 v7.l, v7.l, -1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 4, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, 8, v5.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v9.h
-; GFX1200-NEXT: v_and_b16 v7.l, v9.l, v7.l
-; GFX1200-NEXT: v_lshlrev_b16 v10.h, 2, v4.l
-; GFX1200-NEXT: v_or_b16 v8.l, v8.h, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, 7, v6.h
+; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 4, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, vcc_lo
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v5.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v8.h
+; GFX1200-NEXT: v_and_b16 v7.l, v8.l, v7.l
+; GFX1200-NEXT: v_or_b16 v6.h, v7.h, v6.h
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, 14
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, 7, v4.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v7.l
-; GFX1200-NEXT: v_and_b16 v7.l, v5.h, 1
-; GFX1200-NEXT: v_or_b16 v10.h, v8.h, v10.h
-; GFX1200-NEXT: v_lshrrev_b16 v9.h, v4.h, v9.l
-; GFX1200-NEXT: v_or_b16 v3.l, v8.l, v3.l
+; GFX1200-NEXT: v_and_b16 v7.l, v5.l, 1
+; GFX1200-NEXT: v_or_b16 v3.h, v6.h, v3.h
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, 2, v6.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, v4.h, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s1
+; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v7.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc_lo
-; GFX1200-NEXT: v_or_b16 v7.l, v11.l, v7.l
-; GFX1200-NEXT: v_or_b16 v8.l, v10.h, v5.l
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v4.l
-; GFX1200-NEXT: v_and_b16 v11.h, v9.h, 1
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v2.l, v2.l
-; GFX1200-NEXT: v_and_b16 v6.h, v6.h, v7.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.l, v7.h, v9.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v9.l, v7.h, v9.l
+; GFX1200-NEXT: v_and_b16 v9.h, v8.h, 1
+; GFX1200-NEXT: v_cndmask_b16 v2.h, 0x7e, v2.h, s0
+; GFX1200-NEXT: v_and_b16 v4.l, v4.l, v6.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v3.l
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v6.l
+; GFX1200-NEXT: v_or_b16 v7.l, v9.l, v3.l
+; GFX1200-NEXT: v_or_b16 v9.l, v11.l, v9.h
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v5.l, v4.l
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, v5.h, v8.l
+; GFX1200-NEXT: s_and_b32 s1, s1, s0
+; GFX1200-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v6.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v4.l
+; GFX1200-NEXT: v_and_b16 v3.l, v5.h, v9.l
+; GFX1200-NEXT: v_or_b16 v4.h, 0x7c, v7.h
+; GFX1200-NEXT: v_and_b16 v2.h, 0xff, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v8.l, v3.l, vcc_lo
-; GFX1200-NEXT: v_or_b16 v10.h, v12.l, v11.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v5.h, v6.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v5.l
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s2, 30, v4.l
-; GFX1200-NEXT: v_and_b16 v5.h, v7.l, v10.h
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v10.l, s5
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s3, 3, v3.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v1.h
-; GFX1200-NEXT: s_and_b32 s1, s2, s1
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v5.h, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v4.l
-; GFX1200-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v1.h
-; GFX1200-NEXT: v_or_b16 v4.l, 0x7c, v8.h
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v9.h, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v7.l, v3.h, vcc_lo
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s2
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.h, v13.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s0
+; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, v5.l
+; GFX1200-NEXT: v_add_nc_u16 v3.l, v8.h, v3.l
+; GFX1200-NEXT: v_sub_nc_u16 v6.l, -5, v5.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_or_b32 s1, vcc_lo, s1
-; GFX1200-NEXT: v_mov_b16_e32 v5.l, v7.l
-; GFX1200-NEXT: v_sub_nc_u16 v7.l, -5, v5.h
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7e, v3.h, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v5.l, 8, v1.l
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v4.l, s1
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v6.l, v5.l
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v1.l
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v2.h
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.h, s1
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v3.l
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v12.l
+; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, 14
+; GFX1200-NEXT: v_min_u16 v6.l, v6.l, 15
+; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v5.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v6.h, 0, 4, vcc_lo
-; GFX1200-NEXT: v_min_u16 v7.l, v7.l, 15
-; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 14
-; GFX1200-NEXT: v_and_b16 v6.l, 0x80, v6.l
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v8.h, s0
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s3
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, 2, v5.l
-; GFX1200-NEXT: v_or_b16 v6.h, v6.l, v6.h
-; GFX1200-NEXT: v_sub_nc_u16 v8.h, v7.l, 1 clamp
-; GFX1200-NEXT: v_cmp_class_f16_e64 s2, v2.h, 0x204
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v5.l
-; GFX1200-NEXT: v_or_b16 v7.h, v6.l, v7.h
-; GFX1200-NEXT: v_or_b16 v4.h, v6.h, v4.h
-; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.h, 1
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, v4.l, s2
-; GFX1200-NEXT: v_or_b16 v6.h, v7.h, v3.l
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v7.h, s1
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 2, v2.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v7.h, v13.l
+; GFX1200-NEXT: v_sub_nc_u16 v8.l, v6.l, 1 clamp
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, vcc_lo
+; GFX1200-NEXT: v_or_b16 v6.h, v5.l, v6.h
+; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v12.l, 0x204
+; GFX1200-NEXT: v_and_b16 v8.h, 0x3ff, v7.h
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.l, 1
+; GFX1200-NEXT: v_or_b16 v3.l, v6.h, v3.l
+; GFX1200-NEXT: v_or_b16 v6.h, v7.l, v4.l
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v2.l
+; GFX1200-NEXT: v_or_b16 v7.l, 0x400, v8.h
; GFX1200-NEXT: v_add_nc_u16 v9.l, v9.l, -1
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v5.l
-; GFX1200-NEXT: v_or_b16 v9.h, 0x7c, v6.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, v4.h, s0
+; GFX1200-NEXT: v_and_b16 v4.h, 0x7f, v7.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v4.l, v6.h, v4.h, vcc_lo
-; GFX1200-NEXT: v_or_b16 v3.l, 0x400, v7.h
-; GFX1200-NEXT: v_and_b16 v4.h, 0x7f, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v7.h
-; GFX1200-NEXT: s_and_b32 s0, s1, s0
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v7.l
-; GFX1200-NEXT: v_and_b16 v6.h, v3.l, v9.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v6.h, v3.l, vcc_lo
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v8.h
+; GFX1200-NEXT: v_and_b16 v6.h, v7.l, v9.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.h, 7, v7.h
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
-; GFX1200-NEXT: v_and_b16 v4.h, v7.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, v7.l, v3.l
-; GFX1200-NEXT: v_lshrrev_b16 v3.l, v8.h, v3.l
-; GFX1200-NEXT: v_and_b16 v3.h, 0xff, v3.h
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v6.l, v7.l
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v2.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v2.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.h
-; GFX1200-NEXT: v_and_b16 v6.h, v9.l, 1
-; GFX1200-NEXT: v_cmp_class_f16_e64 s5, v1.h, 0x204
+; GFX1200-NEXT: v_and_b16 v6.h, v8.h, 1
+; GFX1200-NEXT: v_lshrrev_b32_e32 v12, 16, v0
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b16 v3.h, 0x7e, v3.h, s0
+; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v1.l, 0x204
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v5.l
-; GFX1200-NEXT: v_mov_b16_e32 v5.l, v10.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v11.l
+; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.l
+; GFX1200-NEXT: v_and_b16 v4.l, v4.h, 1
+; GFX1200-NEXT: v_or_b16 v6.h, v9.l, v6.h
+; GFX1200-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
+; GFX1200-NEXT: v_cmp_class_f16_e64 s5, v13.l, 0x204
+; GFX1200-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX1200-NEXT: v_or_b16 v2.l, v10.l, v4.l
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, v8.l, v7.l
+; GFX1200-NEXT: v_and_b16 v6.h, v7.h, v6.h
+; GFX1200-NEXT: v_or_b16 v7.l, 0x7c, v5.l
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1200-NEXT: v_or_b16 v4.h, v5.l, v4.h
-; GFX1200-NEXT: v_lshrrev_b16 v5.l, 7, v8.l
-; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v2.h, v2.h
+; GFX1200-NEXT: s_or_b32 s1, s2, s1
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v6.l
+; GFX1200-NEXT: v_and_b16 v2.l, v4.l, v2.l
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v8.h, v6.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v4.l, v9.h, s0
-; GFX1200-NEXT: v_or_b16 v6.h, v10.l, v6.h
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v1.l
-; GFX1200-NEXT: v_and_b16 v4.l, v5.l, v4.h
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v4.h, v0.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s1
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s1, 0, v1.l
+; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v3.h
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7e, v2.l, vcc_lo
-; GFX1200-NEXT: v_and_b16 v3.l, v3.l, v6.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v6.l, s0
-; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.h, v4.l
-; GFX1200-NEXT: v_sub_nc_u16 v5.l, -5, v4.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v1
-; GFX1200-NEXT: v_cndmask_b16 v3.l, 0, v3.l, s1
-; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v1.l, 0x204
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, v2.l, vcc_lo
; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.l
-; GFX1200-NEXT: v_min_u16 v5.l, v5.l, 15
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v9.l, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v9.h, s0
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v5.l, s1
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v5.l, v0.l
+; GFX1200-NEXT: v_add_nc_u16 v2.l, v4.h, v2.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
-; GFX1200-NEXT: v_sub_nc_u16 v6.h, v5.l, 1 clamp
; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.l, v1.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v3.l
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 7, v7.l
+; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, s0
+; GFX1200-NEXT: v_sub_nc_u16 v4.h, -5, v5.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 31, v1
; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, v6.l
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v6.l, v0.l
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, v6.h, 1
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v2.l
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, vcc_lo
+; GFX1200-NEXT: v_min_u16 v4.h, v4.h, 15
+; GFX1200-NEXT: v_lshlrev_b16 v6.h, 7, v7.l
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v5.h, 14
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v5.h, v0.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, 0, 4, s0
-; GFX1200-NEXT: v_cndmask_b16 v9.l, v4.l, 0, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v5.h, 14
-; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v6.l
+; GFX1200-NEXT: v_sub_nc_u16 v6.l, v4.h, 1 clamp
+; GFX1200-NEXT: v_cndmask_b16 v2.l, v2.l, 0, s0
+; GFX1200-NEXT: v_lshlrev_b16 v7.l, 2, v1.h
+; GFX1200-NEXT: v_and_b16 v7.h, 0x3ff, v5.h
+; GFX1200-NEXT: v_or_b16 v1.l, v6.h, v1.l
+; GFX1200-NEXT: v_lshlrev_b16 v8.l, v6.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v3.l, 0x7e, v3.l, s1
+; GFX1200-NEXT: v_or_b16 v7.l, v6.h, v7.l
+; GFX1200-NEXT: v_or_b16 v8.h, 0x400, v7.h
+; GFX1200-NEXT: v_or_b16 v1.l, v1.l, v2.l
; GFX1200-NEXT: v_add_nc_u16 v8.l, v8.l, -1
-; GFX1200-NEXT: v_or_b16 v4.l, v3.h, v2.l
-; GFX1200-NEXT: v_cndmask_b16 v2.l, 0x7e, v2.h, s1
-; GFX1200-NEXT: v_lshlrev_b16 v8.h, 2, v5.h
-; GFX1200-NEXT: v_or_b16 v9.h, 0x400, v7.h
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, 0, s0
-; GFX1200-NEXT: v_or_b16 v1.l, v7.l, v1.l
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v9.l
-; GFX1200-NEXT: v_or_b16 v8.h, v7.l, v8.h
-; GFX1200-NEXT: v_and_b16 v2.h, v9.h, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v3.h, v5.l, v9.h
-; GFX1200-NEXT: v_or_b16 v1.l, v1.l, v3.l
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s0, 30, v5.h
-; GFX1200-NEXT: v_or_b16 v3.l, v8.h, v9.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v2.h
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v5.h
-; GFX1200-NEXT: v_and_b16 v2.h, v3.h, 1
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.l
+; GFX1200-NEXT: v_or_b16 v2.l, v7.l, v4.l
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s0, 30, v1.h
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v1.h
+; GFX1200-NEXT: v_and_b16 v3.h, v8.h, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v4.l, v4.h, v8.h
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, v6.l, v8.h
; GFX1200-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX1200-NEXT: v_lshrrev_b16 v6.h, v6.h, v9.h
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, s1
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.h
-; GFX1200-NEXT: v_and_b16 v5.h, 0x7f, v6.l
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v7.h
-; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.h, v0.h
-; GFX1200-NEXT: v_or_b16 v2.h, v8.l, v2.h
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
-; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v3.l, v0.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.h
-; GFX1200-NEXT: v_and_b16 v5.h, v7.h, 1
-; GFX1200-NEXT: v_and_b16 v2.h, v6.h, v2.h
-; GFX1200-NEXT: v_lshrrev_b16 v6.l, 7, v6.l
-; GFX1200-NEXT: v_sub_nc_u16 v8.l, -5, v3.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 30, v1.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v3.h
+; GFX1200-NEXT: v_and_b16 v1.h, v4.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v2.l, v1.l, s1
+; GFX1200-NEXT: v_frexp_exp_i16_f16_e32 v2.l, v12.l
+; GFX1200-NEXT: v_and_b16 v3.h, 0x7f, v5.h
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s2
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: s_or_b32 s0, s2, s0
-; GFX1200-NEXT: v_or_b16 v6.h, 0x7c, v7.l
-; GFX1200-NEXT: v_min_u16 v5.l, v8.l, 15
-; GFX1200-NEXT: v_mov_b16_e32 v8.l, v9.l
+; GFX1200-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1200-NEXT: v_lshrrev_b16 v5.h, 7, v5.h
+; GFX1200-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h
+; GFX1200-NEXT: v_mov_b16_e32 v7.l, v9.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, 0, v2.h, vcc_lo
-; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v1.h
+; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_or_b16 v1.h, v7.l, v1.h
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, 8, v7.h
+; GFX1200-NEXT: v_sub_nc_u16 v7.h, -5, v2.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v4.h
+; GFX1200-NEXT: v_and_b16 v1.h, v6.l, v1.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b16 v4.h, v7.l, 1
+; GFX1200-NEXT: v_min_u16 v3.h, v7.h, 15
+; GFX1200-NEXT: v_mov_b16_e32 v6.l, v8.l
+; GFX1200-NEXT: v_frexp_mant_f16_e32 v8.l, v12.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.h, 0, v1.h, vcc_lo
+; GFX1200-NEXT: v_or_b16 v7.h, 0x7c, v6.h
+; GFX1200-NEXT: v_sub_nc_u16 v8.h, v3.h, 1 clamp
+; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v4.h
+; GFX1200-NEXT: v_and_b16 v6.l, 0x3ff, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v1.h, v4.l, v1.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.h, s0
-; GFX1200-NEXT: v_sub_nc_u16 v9.l, v5.l, 1 clamp
-; GFX1200-NEXT: v_or_b16 v5.h, v8.l, v5.h
-; GFX1200-NEXT: v_and_b16 v8.l, 0x3ff, v8.h
-; GFX1200-NEXT: v_add_nc_u16 v2.h, v3.h, v2.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.l, vcc_lo
-; GFX1200-NEXT: v_lshlrev_b16 v9.h, v9.l, 1
-; GFX1200-NEXT: v_and_b16 v3.h, v6.l, v5.h
-; GFX1200-NEXT: v_or_b16 v5.h, 0x400, v8.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v2.h
-; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v5.l
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v9.h, -1
-; GFX1200-NEXT: v_and_b16 v9.h, 0x7f, v8.h
-; GFX1200-NEXT: v_add_nc_u16 v3.h, v7.h, v3.h
-; GFX1200-NEXT: v_lshrrev_b16 v7.h, 8, v8.l
-; GFX1200-NEXT: v_lshrrev_b16 v8.l, v5.l, v5.h
-; GFX1200-NEXT: v_and_b16 v6.l, v5.h, v6.l
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.h
-; GFX1200-NEXT: v_lshrrev_b16 v8.h, 7, v8.h
-; GFX1200-NEXT: v_and_b16 v9.h, v7.h, 1
-; GFX1200-NEXT: v_lshrrev_b16 v5.h, v9.l, v5.h
-; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v6.l
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.h, s0
+; GFX1200-NEXT: v_lshlrev_b16 v9.l, v8.h, 1
+; GFX1200-NEXT: v_and_b16 v4.l, v5.h, v4.h
+; GFX1200-NEXT: v_or_b16 v4.h, 0x400, v6.l
+; GFX1200-NEXT: v_lshrrev_b16 v6.l, 8, v6.l
+; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v13.l
+; GFX1200-NEXT: v_add_nc_u16 v5.h, v9.l, -1
+; GFX1200-NEXT: v_and_b16 v9.l, 0x7f, v8.l
+; GFX1200-NEXT: v_lshrrev_b16 v8.l, 7, v8.l
+; GFX1200-NEXT: v_add_nc_u16 v4.l, v7.l, v4.l
+; GFX1200-NEXT: v_lshrrev_b16 v7.l, v3.h, v4.h
+; GFX1200-NEXT: v_and_b16 v5.h, v4.h, v5.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s1, 0, v9.l
+; GFX1200-NEXT: v_and_b16 v9.l, v6.l, 1
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.h, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v1.h
+; GFX1200-NEXT: v_cmp_ne_u16_e64 s2, 0, v5.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v10, 0, 1, s1
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v3.h
-; GFX1200-NEXT: v_and_b16 v10.h, v8.l, 1
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, 0, s0
-; GFX1200-NEXT: v_cndmask_b32_e64 v12, 0, 1, s2
-; GFX1200-NEXT: v_or_b16 v9.h, v10.l, v9.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s1, 3, v4.l
+; GFX1200-NEXT: v_and_b16 v5.h, v7.l, 1
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, 0, s0
+; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, s2
+; GFX1200-NEXT: v_or_b16 v9.l, v10.l, v9.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v11, 0, 1, s1
-; GFX1200-NEXT: v_lshrrev_b16 v9.l, 8, v0.l
-; GFX1200-NEXT: v_cndmask_b16 v3.h, v3.h, 0, s1
-; GFX1200-NEXT: v_mov_b16_e32 v10.l, v12.l
-; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v6.h, s5
-; GFX1200-NEXT: v_mov_b16_e32 v6.l, v11.l
-; GFX1200-NEXT: v_and_b16 v5.l, 0x80, v9.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v3.h
-; GFX1200-NEXT: v_or_b16 v7.l, v10.l, v10.h
-; GFX1200-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, v6.l
-; GFX1200-NEXT: v_and_b16 v6.l, v8.h, v9.h
-; GFX1200-NEXT: v_cndmask_b16 v8.h, 0, 4, s0
-; GFX1200-NEXT: v_and_b16 v5.h, v5.h, v7.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-NEXT: v_add_nc_u16 v4.h, v4.h, 14
-; GFX1200-NEXT: v_add_nc_u16 v6.l, v7.h, v6.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_or_b16 v7.h, v5.l, v8.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v5.h, 0, v5.h, vcc_lo
-; GFX1200-NEXT: v_lshlrev_b16 v7.l, 2, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v4.h, v8.h, v4.h
+; GFX1200-NEXT: v_lshrrev_b16 v8.h, 8, v0.l
+; GFX1200-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX1200-NEXT: v_and_b16 v6.h, v8.l, v9.l
+; GFX1200-NEXT: v_cndmask_b16 v8.l, 0, 4, s0
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, v11.l
+; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v3.h
+; GFX1200-NEXT: v_or_b16 v5.h, v10.l, v5.h
+; GFX1200-NEXT: v_add_nc_u16 v6.l, v6.l, v6.h
+; GFX1200-NEXT: v_and_b16 v3.h, 0x80, v8.h
+; GFX1200-NEXT: v_add_nc_u16 v5.l, v5.l, 14
+; GFX1200-NEXT: v_cndmask_b16 v4.l, v4.l, 0, s1
+; GFX1200-NEXT: v_and_b16 v4.h, v4.h, v5.h
; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v6.l
-; GFX1200-NEXT: v_or_b16 v2.h, v7.h, v2.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_add_nc_u16 v5.h, v8.l, v5.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0
-; GFX1200-NEXT: v_or_b16 v7.l, v5.l, v7.l
+; GFX1200-NEXT: v_or_b16 v6.h, v3.h, v8.l
+; GFX1200-NEXT: v_lshlrev_b16 v5.h, 2, v5.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 3, v4.l
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v4.h, 0, v4.h, vcc_lo
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s3, 30, v4.h
-; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v5.h
-; GFX1200-NEXT: v_lshlrev_b16 v7.h, 7, v8.l
-; GFX1200-NEXT: v_or_b16 v7.l, v7.l, v3.h
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, v9.l
-; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v4.h
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.h, 0, 4, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 30, v4.h
-; GFX1200-NEXT: v_cndmask_b16 v4.h, v5.h, 0, vcc_lo
-; GFX1200-NEXT: v_add_nc_u16 v3.l, v3.l, 14
-; GFX1200-NEXT: v_cndmask_b16 v5.h, v6.l, 0, s0
-; GFX1200-NEXT: v_or_b16 v3.h, v7.h, v3.h
+; GFX1200-NEXT: v_or_b16 v1.h, v6.h, v1.h
+; GFX1200-NEXT: v_or_b16 v5.h, v3.h, v5.h
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s3, 30, v5.l
+; GFX1200-NEXT: v_add_nc_u16 v4.h, v7.l, v4.h
+; GFX1200-NEXT: v_mov_b16_e32 v8.l, v9.l
+; GFX1200-NEXT: v_cmp_gt_i16_e64 s1, 1, v5.l
+; GFX1200-NEXT: v_or_b16 v5.h, v5.h, v4.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s4, 30, v5.l
+; GFX1200-NEXT: v_cmp_lt_i16_e32 vcc_lo, 3, v4.h
+; GFX1200-NEXT: v_add_nc_u16 v2.l, v2.l, v8.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v8, 31, v0
+; GFX1200-NEXT: v_cndmask_b16 v5.l, v6.l, 0, s0
; GFX1200-NEXT: s_and_b32 s0, s3, s2
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v7.l, v2.h, s1
-; GFX1200-NEXT: v_lshlrev_b16 v8.l, 2, v3.l
-; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v3.l
-; GFX1200-NEXT: v_or_b16 v3.h, v3.h, v4.h
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b16 v4.h, v4.h, 0, vcc_lo
+; GFX1200-NEXT: v_add_nc_u16 v0.h, v2.l, 14
+; GFX1200-NEXT: v_cndmask_b16 v2.l, 0, 4, vcc_lo
+; GFX1200-NEXT: v_lshlrev_b16 v4.l, 7, v8.l
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v5.h, v1.h, s1
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_or_b32 s3, s4, s0
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v5.h
-; GFX1200-NEXT: v_or_b16 v6.l, v7.h, v8.l
-; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v3.l
-; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v3.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v5.h
-; GFX1200-NEXT: v_or_b16 v5.h, 0x7c, v5.l
+; GFX1200-NEXT: v_lshlrev_b16 v6.h, 2, v0.h
+; GFX1200-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v0.h
+; GFX1200-NEXT: v_or_b16 v2.l, v4.l, v2.l
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s0, 3, v5.l
+; GFX1200-NEXT: v_cmp_eq_u16_e64 s1, 30, v0.h
+; GFX1200-NEXT: v_or_b16 v6.l, v4.l, v6.h
+; GFX1200-NEXT: v_cmp_lt_i16_e64 s2, 30, v0.h
+; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v4.h
+; GFX1200-NEXT: v_cndmask_b16 v1.l, v1.l, v7.h, s5
; GFX1200-NEXT: s_and_b32 s0, s1, s0
-; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v1.h, v1.h
+; GFX1200-NEXT: v_or_b16 v4.h, v6.l, v5.l
+; GFX1200-NEXT: v_or_b16 v5.l, 0x7c, v3.h
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX1200-NEXT: s_or_b32 s0, s2, s0
+; GFX1200-NEXT: v_cmp_o_f16_e64 s1, v13.l, v13.l
+; GFX1200-NEXT: v_cmp_class_f16_e64 s2, v0.l, 0x204
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v4.h, v3.h, vcc_lo
-; GFX1200-NEXT: v_or_b16 v3.h, 0x7c, v7.h
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v2.h, v5.h, s3
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v4.h, v2.l, vcc_lo
+; GFX1200-NEXT: v_or_b16 v2.l, 0x7c, v4.l
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.l, s3
; GFX1200-NEXT: v_cmp_eq_f16_e32 vcc_lo, 0, v0.l
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.l, s1
-; GFX1200-NEXT: v_cmp_class_f16_e64 s2, v0.l, 0x204
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v3.l, v3.l, v3.h, s0
-; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v0.h
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v2.l, s0
+; GFX1200-NEXT: v_cmp_eq_f16_e64 s0, 0, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v2.h, v5.l, vcc_lo
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v3.h, vcc_lo
; GFX1200-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.l
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b16 v2.h, v3.l, v7.h, s0
-; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v0.h, 0x204
-; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.h, s2
+; GFX1200-NEXT: v_cndmask_b16 v0.h, v0.h, v4.l, s0
+; GFX1200-NEXT: v_cmp_class_f16_e64 s0, v12.l, 0x204
+; GFX1200-NEXT: v_cndmask_b16 v1.h, v1.h, v5.l, s2
+; GFX1200-NEXT: v_cvt_u32_u16_e32 v4, v2.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-NEXT: v_cndmask_b16 v0.l, v2.h, v3.h, s0
-; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v0.h, v0.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b16 v0.l, v0.h, v2.l, s0
; GFX1200-NEXT: v_lshlrev_b16 v0.h, 8, v1.l
-; GFX1200-NEXT: v_cvt_u32_u16_e32 v3, v4.l
+; GFX1200-NEXT: v_cmp_o_f16_e64 s0, v12.l, v12.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b16 v1.l, 0x7e, v1.h, vcc_lo
+; GFX1200-NEXT: v_or_b32_e32 v4, 0x7e7e0000, v4
+; GFX1200-NEXT: v_or_b16 v2.l, v3.l, v0.h
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b16 v0.l, 0x7e, v0.l, s0
-; GFX1200-NEXT: v_or_b16 v2.l, v2.l, v0.h
-; GFX1200-NEXT: v_or_b32_e32 v6, 0x7e7e0000, v3
; GFX1200-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v0.l
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_lshrrev_b64 v[3:4], 24, v[3:4]
; GFX1200-NEXT: v_or_b16 v0.l, v0.h, v1.l
-; GFX1200-NEXT: v_lshrrev_b64 v[5:6], 24, v[5:6]
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX1200-NEXT: v_mov_b16_e32 v4.l, v2.h
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-TRUE16-LABEL: to_bf8_v5f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index 365c938d8f6e6..0b73030d81a45 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -407,13 +407,12 @@ define i32 @test_D139469_f16(half %arg) {
; GFX11-SDAG-TRUE16-LABEL: test_D139469_f16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %bb
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x291e
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_fmaak_f16 v0.l, v0.l, v1.l, 0x211e
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.h, 0x291e, v1.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_min_f16_e32 v0.l, v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x291e
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v1.l, 0x291e, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_fmaak_f16 v0.l, v0.l, v0.h, 0x211e
+; GFX11-SDAG-TRUE16-NEXT: v_min_f16_e32 v0.l, v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -465,13 +464,12 @@ define i32 @test_D139469_f16(half %arg) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x291e
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_fmaak_f16 v0.l, v0.l, v1.l, 0x211e
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.h, 0x291e, v1.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_min_num_f16_e32 v0.l, v0.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x291e
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v1.l, 0x291e, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_fmaak_f16 v0.l, v0.l, v0.h, 0x211e
+; GFX12-SDAG-TRUE16-NEXT: v_min_num_f16_e32 v0.l, v1.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, 0, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
index 232fbdb370c6a..b7f72c58c5264 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
@@ -1239,9 +1239,12 @@ define <2 x half> @v_max3_v2f16_maximumnum_maximumnum__v_v_v_0(<2 x half> %a, <2
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v1
+; GFX12-NEXT: v_pk_max_num_f16 v1, v2, v2
; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
%tmp0 = call <2 x half> @llvm.maximumnum.v2f16(<2 x half> %a, <2 x half> %b)
%max3 = call <2 x half> @llvm.maximumnum.v2f16(<2 x half> %tmp0, <2 x half> %c)
@@ -1408,11 +1411,18 @@ define <3 x half> @v_max3_v3f16_maximumnum_maximumnum__v_v_v_0(<3 x half> %a, <3
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
+; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v3
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v4, v5, v5
; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v4
-; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v2
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
%tmp0 = call <3 x half> @llvm.maximumnum.v3f16(<3 x half> %a, <3 x half> %b)
%max3 = call <3 x half> @llvm.maximumnum.v3f16(<3 x half> %tmp0, <3 x half> %c)
@@ -1611,11 +1621,18 @@ define <4 x half> @v_max3_v4f16_maximumnum_maximumnum__v_v_v_0(<4 x half> %a, <4
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
+; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v3
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v4, v5, v5
; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v4
-; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v2
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
%tmp0 = call <4 x half> @llvm.maximumnum.v4f16(<4 x half> %a, <4 x half> %b)
%max3 = call <4 x half> @llvm.maximumnum.v4f16(<4 x half> %tmp0, <4 x half> %c)
@@ -1701,8 +1718,11 @@ define double @v_max3_f64_maximumnum_maximumnum__v_v_v_0(double %a, double %b, d
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_setpc_b64 s[30:31]
%tmp0 = call double @llvm.maximumnum.f64(double %a, double %b)
@@ -1889,47 +1909,49 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v4
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -1982,17 +2004,19 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
@@ -2001,35 +2025,35 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
@@ -2427,57 +2451,59 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v6.l, v4.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, s1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v7
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v8
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v6, v8
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v2.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v5.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.l, v1.l, s1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
@@ -2589,51 +2615,53 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v6.l, v4.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s2
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s1
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v6, v7
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v8
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v6, v8
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
@@ -2641,11 +2669,11 @@ define <2 x bfloat> @v_max3_v2bf16_maximumnum_maximumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v2.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v5.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.l, v1.l, s1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
@@ -3110,7 +3138,11 @@ define float @v_min3_f32_maximumnum_neg_maximumnum__v_v_v_0(float %a, float %b,
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_minmax_num_f32 v0, v0, -v1, v2
+; GFX12-NEXT: v_max_num_f32_e64 v1, -v1, -v1
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_minmax_num_f32 v0, v0, v1, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg float %a
%tmp0 = call float @llvm.maximumnum.f32(float %neg.a, float %b)
@@ -3194,9 +3226,13 @@ define float @v_min3_f32_maximumnum_fabs_maximumnum__v_v_v_0(float %a, float %b,
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_max_num_f32_e64 v0, -v0, v1
+; GFX12-NEXT: v_max_num_f32_e32 v1, v1, v1
+; GFX12-NEXT: v_max_num_f32_e64 v0, -v0, -v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_max_num_f32 v0, v0, v1 :: v_dual_max_num_f32 v1, v2, v2
+; GFX12-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e64 v0, |v0|, v2
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v1
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg float %a
%tmp0 = call float @llvm.maximumnum.f32(float %neg.a, float %b)
@@ -3280,9 +3316,13 @@ define float @v_min3_f32_maximumnum_fneg_fabs_maximumnum__v_v_v_0(float %a, floa
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_max_num_f32_e64 v0, -v0, v1
+; GFX12-NEXT: v_max_num_f32_e32 v1, v1, v1
+; GFX12-NEXT: v_max_num_f32_e64 v0, -v0, -v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_max_num_f32 v0, v0, v1 :: v_dual_max_num_f32 v1, v2, v2
+; GFX12-NEXT: v_or_b32_e32 v0, 0x80000000, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e64 v0, -|v0|, v2
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v1
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg float %a
%tmp0 = call float @llvm.maximumnum.f32(float %neg.a, float %b)
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index c618573b74734..375671be0a9a0 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -485,26 +485,26 @@ define amdgpu_ps half @test_fmaximum_f16_ss(half inreg %a, half inreg %b) {
; GFX1170-GISEL-FAKE16-NEXT: v_maximum_f16 v0, s0, s1
; GFX1170-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-TRUE16-LABEL: test_fmaximum_f16_ss:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: s_maximum_f16 s0, s0, s1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fmaximum_f16_ss:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_maximum_f16 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-FAKE16-LABEL: test_fmaximum_f16_ss:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: s_maximum_f16 s0, s0, s1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: test_fmaximum_f16_ss:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: s_maximum_f16 s0, s0, s1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: test_fmaximum_f16_ss:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_maximum_f16 s0, s0, s1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-FAKE16-LABEL: test_fmaximum_f16_ss:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: s_maximum_f16 s0, s0, s1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
%val = call half @llvm.maximum.f16(half %a, half %b)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
index 5e7bd87ecfafb..cfb35a91503c1 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
@@ -2416,9 +2416,9 @@ define half @v_fmaximum3_f16_const1_const2(half %a) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4800
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4800
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_maximum3_f16 v0.l, v0.l, v1.l, 0x4c00
+; GFX12-TRUE16-NEXT: v_maximum3_f16 v0.l, v0.l, v0.h, 0x4c00
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_fmaximum3_f16_const1_const2:
@@ -2436,9 +2436,9 @@ define half @v_fmaximum3_f16_const1_const2(half %a) {
; GFX1170-TRUE16-LABEL: v_fmaximum3_f16_const1_const2:
; GFX1170-TRUE16: ; %bb.0:
; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4800
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4800
; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-TRUE16-NEXT: v_maximum3_f16 v0.l, v0.l, v1.l, 0x4c00
+; GFX1170-TRUE16-NEXT: v_maximum3_f16 v0.l, v0.l, v0.h, 0x4c00
; GFX1170-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-FAKE16-LABEL: v_fmaximum3_f16_const1_const2:
diff --git a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
index 28c9cedb2fa32..4b015bea4dcbd 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
@@ -62,11 +62,12 @@ define amdgpu_kernel void @test_fmax_f32_ieee_mode_on(ptr addrspace(1) %out, flo
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_max_num_f32 s2, s2, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s3, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s2, s2
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -184,10 +185,12 @@ define amdgpu_kernel void @test_fmax_v2f32(ptr addrspace(1) %out, <2 x float> %a
; GFX12-SDAG-NEXT: s_mov_b32 s7, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s6, -1
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_max_num_f32 s0, s0, s2
-; GFX12-SDAG-NEXT: s_max_num_f32 s1, s1, s3
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s3, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s1, s1
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v3, s0, s0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v1, v1, v0 :: v_dual_max_num_f32 v0, v3, v2
; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -303,15 +306,18 @@ define amdgpu_kernel void @test_fmax_v3f32(ptr addrspace(1) %out, <3 x float> %a
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_max_num_f32 s2, s8, s12
-; GFX12-SDAG-NEXT: s_max_num_f32 s3, s9, s13
-; GFX12-SDAG-NEXT: s_max_num_f32 s4, s10, s14
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(NEXT) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s14, s14
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s10, s10
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v3, s13, s13
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s9, s9
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v5, s12, s12
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v6, s8, s8
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v2, v1, v0 :: v_dual_max_num_f32 v1, v4, v3
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v0, v6, v5
; GFX12-SDAG-NEXT: buffer_store_b96 v[0:2], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -442,16 +448,20 @@ define amdgpu_kernel void @test_fmax_v4f32(ptr addrspace(1) %out, <4 x float> %a
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_max_num_f32 s2, s8, s12
-; GFX12-SDAG-NEXT: s_max_num_f32 s3, s9, s13
-; GFX12-SDAG-NEXT: s_max_num_f32 s4, s10, s14
-; GFX12-SDAG-NEXT: s_max_num_f32 s5, s11, s15
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s15, s15
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s11, s11
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s14, s14
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s10, s10
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v5, s13, s13
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v6, s9, s9
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v7, s12, s12
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s8, s8
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v3, v1, v0 :: v_dual_max_num_f32 v2, v4, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v1, v6, v5 :: v_dual_max_num_f32 v0, v8, v7
; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -639,24 +649,33 @@ define amdgpu_kernel void @test_fmax_v8f32(ptr addrspace(1) %out, <8 x float> %a
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b512 s[8:23], s[4:5], 0x44
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_max_num_f32 s4, s9, s17
-; GFX12-SDAG-NEXT: s_max_num_f32 s5, s8, s16
-; GFX12-SDAG-NEXT: s_max_num_f32 s6, s12, s20
-; GFX12-SDAG-NEXT: s_max_num_f32 s7, s13, s21
-; GFX12-SDAG-NEXT: s_max_num_f32 s8, s14, s22
-; GFX12-SDAG-NEXT: s_max_num_f32 s9, s15, s23
-; GFX12-SDAG-NEXT: s_max_num_f32 s2, s11, s19
-; GFX12-SDAG-NEXT: s_max_num_f32 s3, s10, s18
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v3, s9
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, s5 :: v_dual_mov_b32 v5, s4
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s2
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s19, s19
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s11, s11
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s18, s18
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s10, s10
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v5, s17, s17
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v6, s9, s9
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v7, s23, s23
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v10, s15, s15
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v11, s22, s22
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v12, s14, s14
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v13, s21, s21
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v14, s13, s13
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v15, s20, s20
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v16, s12, s12
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s16, s16
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v9, s8, s8
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v3, v1, v0 :: v_dual_max_num_f32 v2, v4, v2
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v1, v6, v5 :: v_dual_max_num_f32 v6, v12, v11
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v7, v10, v7 :: v_dual_max_num_f32 v0, v9, v8
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v5, v14, v13 :: v_dual_max_num_f32 v4, v16, v15
; GFX12-SDAG-NEXT: s_clause 0x1
-; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null offset:16
-; GFX12-SDAG-NEXT: buffer_store_b128 v[4:7], off, s[0:3], null
+; GFX12-SDAG-NEXT: buffer_store_b128 v[4:7], off, s[0:3], null offset:16
+; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: test_fmax_v8f32:
@@ -973,42 +992,63 @@ define amdgpu_kernel void @test_fmax_v16f32(ptr addrspace(1) %out, <16 x float>
; GFX12-SDAG-LABEL: test_fmax_v16f32:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x2
-; GFX12-SDAG-NEXT: s_load_b512 s[8:23], s[4:5], 0x64
; GFX12-SDAG-NEXT: s_load_b512 s[36:51], s[4:5], 0xa4
+; GFX12-SDAG-NEXT: s_load_b512 s[8:23], s[4:5], 0x64
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_max_num_f32 s4, s11, s39
-; GFX12-SDAG-NEXT: s_max_num_f32 s5, s10, s38
-; GFX12-SDAG-NEXT: s_max_num_f32 s6, s9, s37
-; GFX12-SDAG-NEXT: s_max_num_f32 s7, s8, s36
-; GFX12-SDAG-NEXT: s_max_num_f32 s8, s15, s43
-; GFX12-SDAG-NEXT: s_max_num_f32 s9, s14, s42
-; GFX12-SDAG-NEXT: s_max_num_f32 s10, s13, s41
-; GFX12-SDAG-NEXT: s_max_num_f32 s11, s12, s40
-; GFX12-SDAG-NEXT: s_max_num_f32 s12, s19, s47
-; GFX12-SDAG-NEXT: s_max_num_f32 s13, s18, s46
-; GFX12-SDAG-NEXT: s_max_num_f32 s14, s17, s45
-; GFX12-SDAG-NEXT: s_max_num_f32 s15, s16, s44
-; GFX12-SDAG-NEXT: s_max_num_f32 s16, s20, s48
-; GFX12-SDAG-NEXT: s_max_num_f32 s17, s21, s49
-; GFX12-SDAG-NEXT: s_max_num_f32 s18, s22, s50
-; GFX12-SDAG-NEXT: s_max_num_f32 s19, s23, s51
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v1, s17
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v3, s19
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, s15 :: v_dual_mov_b32 v5, s14
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v6, s13 :: v_dual_mov_b32 v7, s12
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v8, s11 :: v_dual_mov_b32 v9, s10
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v10, s9 :: v_dual_mov_b32 v11, s8
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v12, s7 :: v_dual_mov_b32 v13, s6
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v14, s5 :: v_dual_mov_b32 v15, s4
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s39, s39
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s11, s11
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s38, s38
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s10, s10
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v5, s37, s37
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v6, s9, s9
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v7, s43, s43
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s15, s15
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v9, s42, s42
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v10, s14, s14
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v11, s41, s41
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v12, s13, s13
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v13, s47, s47
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v14, s19, s19
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v3, v1, v0 :: v_dual_max_num_f32 v2, v4, v2
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v7, v8, v7
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s46, s46
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s18, s18
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v1, v6, v5
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v6, v10, v9
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s45, s45
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v9, s17, s17
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v5, v12, v11
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v18, s40, s40
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v19, s12, s12
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v10, v4, v0
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v9, v9, v8
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s51, s51
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s23, s23
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v12, s50, s50
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v20, s49, s49
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v21, s21, s21
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v22, s48, s48
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v23, s20, s20
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v11, v14, v13
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v13, s22, s22
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s44, s44
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v24, s16, s16
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v16, s36, s36
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v17, s8, s8
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v15, v8, v4
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v14, v13, v12
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v13, v21, v20 :: v_dual_max_num_f32 v12, v23, v22
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v8, v24, v0
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v4, v19, v18
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v0, v17, v16
; GFX12-SDAG-NEXT: s_clause 0x3
-; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null offset:48
-; GFX12-SDAG-NEXT: buffer_store_b128 v[4:7], off, s[0:3], null offset:32
-; GFX12-SDAG-NEXT: buffer_store_b128 v[8:11], off, s[0:3], null offset:16
-; GFX12-SDAG-NEXT: buffer_store_b128 v[12:15], off, s[0:3], null
+; GFX12-SDAG-NEXT: buffer_store_b128 v[12:15], off, s[0:3], null offset:48
+; GFX12-SDAG-NEXT: buffer_store_b128 v[8:11], off, s[0:3], null offset:32
+; GFX12-SDAG-NEXT: buffer_store_b128 v[4:7], off, s[0:3], null offset:16
+; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: test_fmax_v16f32:
@@ -1792,6 +1832,10 @@ define <3 x float> @test_func_fmax_v3f32(<3 x float> %a, <3 x float> %b) #0 {
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v3, v3, v3 :: v_dual_max_num_f32 v0, v0, v0
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v4, v4, v4 :: v_dual_max_num_f32 v1, v1, v1
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v5, v5, v5 :: v_dual_max_num_f32 v2, v2, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_dual_max_num_f32 v0, v0, v3 :: v_dual_max_num_f32 v1, v1, v4
; GFX12-SDAG-NEXT: v_max_num_f32_e32 v2, v2, v5
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1876,11 +1920,12 @@ define amdgpu_kernel void @test_fmax_f16_v_ieee_on(ptr addrspace(1) %out, half %
; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshr_b32 s3, s2, 16
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: s_max_num_f16 s2, s2, s3
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.l, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.h, s3, s3
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.h
; GFX12-SDAG-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -1994,11 +2039,12 @@ define amdgpu_kernel void @test_fmax_f16_s_ieee_on(ptr addrspace(1) %out, half i
; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshr_b32 s3, s2, 16
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: s_max_num_f16 s2, s2, s3
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.l, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.h, s3, s3
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_max_num_f16_e32 v0.l, v0.l, v0.h
; GFX12-SDAG-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -2044,14 +2090,14 @@ define amdgpu_ps half @test_fmax_f16_s_ieee_off(half inreg %a, half inreg %b) #0
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_max_num_f16 s0, s0, s1
; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: test_fmax_f16_s_ieee_off:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_max_num_f16 s0, s0, s1
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call half @llvm.maxnum.f16(half %a, half %b)
ret half %val
@@ -2113,11 +2159,12 @@ define amdgpu_kernel void @test_fmax_f32_s_ieee_on(ptr addrspace(1) %out, float
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_max_num_f32 s2, s2, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s3, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s2, s2
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_max_num_f32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -2241,9 +2288,12 @@ define amdgpu_kernel void @test_fmax_v2f16_v_ieee_on(ptr addrspace(1) %out, <2 x
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_pk_max_num_f16 v0, s2, s3
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v0, s3, s3
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v1, s2, s2
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v0, v1, v0
; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -2368,13 +2418,14 @@ define amdgpu_kernel void @test_fmax_v2f16_s_ieee_on(ptr addrspace(1) %out, <2 x
; GFX12-SDAG-LABEL: test_fmax_v2f16_s_ieee_on:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-SDAG-NEXT: s_mov_b32 s6, -1
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_pk_max_num_f16 v0, s2, s3
-; GFX12-SDAG-NEXT: s_mov_b32 s4, s0
-; GFX12-SDAG-NEXT: s_mov_b32 s5, s1
-; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v0, s3, s3
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v1, s2, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v0, v1, v0
+; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: test_fmax_v2f16_s_ieee_on:
@@ -2494,12 +2545,15 @@ define amdgpu_kernel void @test_fmax_f64_v_ieee_on(ptr addrspace(1) %out, double
; GFX12-SDAG-LABEL: test_fmax_f64_v_ieee_on:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[0:1], s[2:3], s[4:5]
+; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[0:1], s[6:7], s[6:7]
+; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[2:3], s[2:3], s[2:3]
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[0:1]
; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -2604,12 +2658,15 @@ define amdgpu_kernel void @test_fmax_f64_s_ieee_on(ptr addrspace(1) %out, double
; GFX12-SDAG-LABEL: test_fmax_f64_s_ieee_on:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[0:1], s[2:3], s[4:5]
+; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[0:1], s[6:7], s[6:7]
+; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[2:3], s[2:3], s[2:3]
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_max_num_f64_e32 v[0:1], v[2:3], v[0:1]
; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
index 882944e363d35..785cb88c1a379 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
@@ -69,16 +69,17 @@ define bfloat @v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum(bfloat %a) #1 {
; GFX11-SDAG-TRUE16-LABEL: v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -194,28 +195,29 @@ define <2 x bfloat> @v_test_fmed3_r_i_i_v2bf16_minimumnum_maximumnum(<2 x bfloat
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e64 s0, v2, v2
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v3.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v2
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, 2.0, v3
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0x4000, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v2
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 4.0, v3
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.h, 0x4080, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
%max = call <2 x bfloat> @llvm.maximumnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> splat (bfloat 2.0))
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
index a98623a215fb2..7676c58005d90 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
@@ -1239,9 +1239,12 @@ define <2 x half> @v_min3_v2f16_minimumnum_minimumnum__v_v_v_0(<2 x half> %a, <2
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v1
+; GFX12-NEXT: v_pk_max_num_f16 v1, v2, v2
; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
%tmp0 = call <2 x half> @llvm.minimumnum.v2f16(<2 x half> %a, <2 x half> %b)
%min3 = call <2 x half> @llvm.minimumnum.v2f16(<2 x half> %tmp0, <2 x half> %c)
@@ -1408,11 +1411,18 @@ define <3 x half> @v_min3_v3f16_minimumnum_minimumnum__v_v_v_0(<3 x half> %a, <3
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
+; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v3
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v4, v5, v5
; GFX12-NEXT: v_pk_min_num_f16 v1, v1, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v4
-; GFX12-NEXT: v_pk_min_num_f16 v1, v1, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v2
+; GFX12-NEXT: v_pk_min_num_f16 v1, v1, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
%tmp0 = call <3 x half> @llvm.minimumnum.v3f16(<3 x half> %a, <3 x half> %b)
%min3 = call <3 x half> @llvm.minimumnum.v3f16(<3 x half> %tmp0, <3 x half> %c)
@@ -1611,11 +1621,18 @@ define <4 x half> @v_min3_v4f16_minimumnum_minimumnum__v_v_v_0(<4 x half> %a, <4
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
+; GFX12-NEXT: v_pk_max_num_f16 v0, v0, v0
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v3
+; GFX12-NEXT: v_pk_max_num_f16 v1, v1, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v4, v5, v5
; GFX12-NEXT: v_pk_min_num_f16 v1, v1, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v4
-; GFX12-NEXT: v_pk_min_num_f16 v1, v1, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_pk_min_num_f16 v0, v0, v2
+; GFX12-NEXT: v_pk_min_num_f16 v1, v1, v4
; GFX12-NEXT: s_setpc_b64 s[30:31]
%tmp0 = call <4 x half> @llvm.minimumnum.v4f16(<4 x half> %a, <4 x half> %b)
%min3 = call <4 x half> @llvm.minimumnum.v4f16(<4 x half> %tmp0, <4 x half> %c)
@@ -1701,8 +1718,11 @@ define double @v_min3_f64_minimumnum_minimumnum__v_v_v_0(double %a, double %b, d
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[0:1], v[0:1], v[0:1]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_min_num_f64_e32 v[0:1], v[0:1], v[4:5]
; GFX12-NEXT: s_setpc_b64 s[30:31]
%tmp0 = call double @llvm.minimumnum.f64(double %a, double %b)
@@ -1891,47 +1911,49 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v4
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -1984,17 +2006,19 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
@@ -2003,35 +2027,35 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v2, v2
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
@@ -2432,57 +2456,59 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v6.l, v4.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, s1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v7
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v8
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v6, v8
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
-; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-SDAG-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v2.h, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v5.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.l, v1.l, s1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
@@ -2594,51 +2620,53 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v1
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v7, v7
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v0.h, v1.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v6.l, v4.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v1.h, v3.l, s2
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s1
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v6, v7
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v8
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v6, v8
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v3.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
-; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s1, s0
-; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-SDAG-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s2, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v5, v5
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v6, v6
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v4.l, v3.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
@@ -2646,11 +2674,11 @@ define <2 x bfloat> @v_min3_v2bf16_minimumnum_minimumnum__v_v_v_0(<2 x bfloat> %
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v2.h, vcc_lo
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v5.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v2.h, v1.l, s1
+; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.l, v1.l, s1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s2
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
@@ -3115,7 +3143,11 @@ define float @v_min3_f32_minimumnum_neg_minimumnum__v_v_v_0(float %a, float %b,
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_maxmin_num_f32 v0, v0, -v1, v2
+; GFX12-NEXT: v_max_num_f32_e64 v1, -v1, -v1
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT: v_max_num_f32_e32 v2, v2, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_maxmin_num_f32 v0, v0, v1, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg float %a
%tmp0 = call float @llvm.minimumnum.f32(float %neg.a, float %b)
@@ -3199,9 +3231,13 @@ define float @v_min3_f32_minimumnum_fabs_minimumnum__v_v_v_0(float %a, float %b,
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_min_num_f32_e64 v0, -v0, v1
+; GFX12-NEXT: v_max_num_f32_e32 v1, v1, v1
+; GFX12-NEXT: v_max_num_f32_e64 v0, -v0, -v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_min_num_f32 v0, v0, v1 :: v_dual_max_num_f32 v1, v2, v2
+; GFX12-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e64 v0, |v0|, v2
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v1
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg float %a
%tmp0 = call float @llvm.minimumnum.f32(float %neg.a, float %b)
@@ -3285,9 +3321,13 @@ define float @v_min3_f32_minimumnum_fneg_fabs_minimumnum__v_v_v_0(float %a, floa
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_min_num_f32_e64 v0, -v0, v1
+; GFX12-NEXT: v_max_num_f32_e32 v1, v1, v1
+; GFX12-NEXT: v_max_num_f32_e64 v0, -v0, -v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_min_num_f32 v0, v0, v1 :: v_dual_max_num_f32 v1, v2, v2
+; GFX12-NEXT: v_or_b32_e32 v0, 0x80000000, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e64 v0, -|v0|, v2
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v1
; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.a = fneg float %a
%tmp0 = call float @llvm.minimumnum.f32(float %neg.a, float %b)
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index b63bc005395a8..2dee6e0fc1c1c 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -485,26 +485,26 @@ define amdgpu_ps half @test_fminimum_f16_ss(half inreg %a, half inreg %b) {
; GFX1170-GISEL-FAKE16-NEXT: v_minimum_f16 v0, s0, s1
; GFX1170-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-TRUE16-LABEL: test_fminimum_f16_ss:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: s_minimum_f16 s0, s0, s1
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: test_fminimum_f16_ss:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_minimum_f16 s0, s0, s1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-FAKE16-LABEL: test_fminimum_f16_ss:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: s_minimum_f16 s0, s0, s1
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: test_fminimum_f16_ss:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: s_minimum_f16 s0, s0, s1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: test_fminimum_f16_ss:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_minimum_f16 s0, s0, s1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-FAKE16-LABEL: test_fminimum_f16_ss:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: s_minimum_f16 s0, s0, s1
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
%val = call half @llvm.minimum.f16(half %a, half %b)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum3.ll b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
index b2d788280db87..5e96ae2bf9360 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
@@ -2416,9 +2416,9 @@ define half @v_fminimum3_f16_const1_const2(half %a) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4800
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4800
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_minimum3_f16 v0.l, v0.l, v1.l, 0x4c00
+; GFX12-TRUE16-NEXT: v_minimum3_f16 v0.l, v0.l, v0.h, 0x4c00
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_fminimum3_f16_const1_const2:
@@ -2436,9 +2436,9 @@ define half @v_fminimum3_f16_const1_const2(half %a) {
; GFX1170-TRUE16-LABEL: v_fminimum3_f16_const1_const2:
; GFX1170-TRUE16: ; %bb.0:
; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x4800
+; GFX1170-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4800
; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-TRUE16-NEXT: v_minimum3_f16 v0.l, v0.l, v1.l, 0x4c00
+; GFX1170-TRUE16-NEXT: v_minimum3_f16 v0.l, v0.l, v0.h, 0x4c00
; GFX1170-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-FAKE16-LABEL: v_fminimum3_f16_const1_const2:
diff --git a/llvm/test/CodeGen/AMDGPU/fminnum.ll b/llvm/test/CodeGen/AMDGPU/fminnum.ll
index eeaa9c9727de8..208aeb5b76b8f 100644
--- a/llvm/test/CodeGen/AMDGPU/fminnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminnum.ll
@@ -62,11 +62,12 @@ define amdgpu_kernel void @test_fmin_f32_ieee_mode_on(ptr addrspace(1) %out, flo
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_min_num_f32 s2, s2, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s3, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s2, s2
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -229,10 +230,12 @@ define amdgpu_kernel void @test_fmin_v2f32(ptr addrspace(1) %out, <2 x float> %a
; GFX12-SDAG-NEXT: s_mov_b32 s7, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s6, -1
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_min_num_f32 s0, s0, s2
-; GFX12-SDAG-NEXT: s_min_num_f32 s1, s1, s3
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s3, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s1, s1
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v3, s0, s0
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_dual_min_num_f32 v1, v1, v0 :: v_dual_min_num_f32 v0, v3, v2
; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -360,16 +363,20 @@ define amdgpu_kernel void @test_fmin_v4f32(ptr addrspace(1) %out, <4 x float> %a
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b256 s[8:15], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_min_num_f32 s2, s8, s12
-; GFX12-SDAG-NEXT: s_min_num_f32 s3, s9, s13
-; GFX12-SDAG-NEXT: s_min_num_f32 s4, s10, s14
-; GFX12-SDAG-NEXT: s_min_num_f32 s5, s11, s15
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s15, s15
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s11, s11
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s14, s14
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s10, s10
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v5, s13, s13
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v6, s9, s9
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v7, s12, s12
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s8, s8
+; GFX12-SDAG-NEXT: v_dual_min_num_f32 v3, v1, v0 :: v_dual_min_num_f32 v2, v4, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-NEXT: v_dual_min_num_f32 v1, v6, v5 :: v_dual_min_num_f32 v0, v8, v7
; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -557,24 +564,33 @@ define amdgpu_kernel void @test_fmin_v8f32(ptr addrspace(1) %out, <8 x float> %a
; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b512 s[8:23], s[4:5], 0x44
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_min_num_f32 s4, s9, s17
-; GFX12-SDAG-NEXT: s_min_num_f32 s5, s8, s16
-; GFX12-SDAG-NEXT: s_min_num_f32 s6, s12, s20
-; GFX12-SDAG-NEXT: s_min_num_f32 s7, s13, s21
-; GFX12-SDAG-NEXT: s_min_num_f32 s8, s14, s22
-; GFX12-SDAG-NEXT: s_min_num_f32 s9, s15, s23
-; GFX12-SDAG-NEXT: s_min_num_f32 s2, s11, s19
-; GFX12-SDAG-NEXT: s_min_num_f32 s3, s10, s18
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v3, s9
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, s5 :: v_dual_mov_b32 v5, s4
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s2
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s19, s19
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s11, s11
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s18, s18
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s10, s10
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v5, s17, s17
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v6, s9, s9
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v7, s23, s23
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v10, s15, s15
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v11, s22, s22
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v12, s14, s14
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v13, s21, s21
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v14, s13, s13
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v15, s20, s20
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v16, s12, s12
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s16, s16
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v9, s8, s8
+; GFX12-SDAG-NEXT: v_dual_min_num_f32 v3, v1, v0 :: v_dual_min_num_f32 v2, v4, v2
+; GFX12-SDAG-NEXT: v_dual_min_num_f32 v1, v6, v5 :: v_dual_min_num_f32 v6, v12, v11
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-SDAG-NEXT: v_dual_min_num_f32 v7, v10, v7 :: v_dual_min_num_f32 v0, v9, v8
+; GFX12-SDAG-NEXT: v_dual_min_num_f32 v5, v14, v13 :: v_dual_min_num_f32 v4, v16, v15
; GFX12-SDAG-NEXT: s_clause 0x1
-; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null offset:16
-; GFX12-SDAG-NEXT: buffer_store_b128 v[4:7], off, s[0:3], null
+; GFX12-SDAG-NEXT: buffer_store_b128 v[4:7], off, s[0:3], null offset:16
+; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: test_fmin_v8f32:
@@ -891,42 +907,63 @@ define amdgpu_kernel void @test_fmin_v16f32(ptr addrspace(1) %out, <16 x float>
; GFX12-SDAG-LABEL: test_fmin_v16f32:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x2
-; GFX12-SDAG-NEXT: s_load_b512 s[8:23], s[4:5], 0x64
; GFX12-SDAG-NEXT: s_load_b512 s[36:51], s[4:5], 0xa4
+; GFX12-SDAG-NEXT: s_load_b512 s[8:23], s[4:5], 0x64
; GFX12-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_min_num_f32 s4, s11, s39
-; GFX12-SDAG-NEXT: s_min_num_f32 s5, s10, s38
-; GFX12-SDAG-NEXT: s_min_num_f32 s6, s9, s37
-; GFX12-SDAG-NEXT: s_min_num_f32 s7, s8, s36
-; GFX12-SDAG-NEXT: s_min_num_f32 s8, s15, s43
-; GFX12-SDAG-NEXT: s_min_num_f32 s9, s14, s42
-; GFX12-SDAG-NEXT: s_min_num_f32 s10, s13, s41
-; GFX12-SDAG-NEXT: s_min_num_f32 s11, s12, s40
-; GFX12-SDAG-NEXT: s_min_num_f32 s12, s19, s47
-; GFX12-SDAG-NEXT: s_min_num_f32 s13, s18, s46
-; GFX12-SDAG-NEXT: s_min_num_f32 s14, s17, s45
-; GFX12-SDAG-NEXT: s_min_num_f32 s15, s16, s44
-; GFX12-SDAG-NEXT: s_min_num_f32 s16, s20, s48
-; GFX12-SDAG-NEXT: s_min_num_f32 s17, s21, s49
-; GFX12-SDAG-NEXT: s_min_num_f32 s18, s22, s50
-; GFX12-SDAG-NEXT: s_min_num_f32 s19, s23, s51
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, s16 :: v_dual_mov_b32 v1, s17
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v3, s19
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v4, s15 :: v_dual_mov_b32 v5, s14
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v6, s13 :: v_dual_mov_b32 v7, s12
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v8, s11 :: v_dual_mov_b32 v9, s10
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v10, s9 :: v_dual_mov_b32 v11, s8
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v12, s7 :: v_dual_mov_b32 v13, s6
-; GFX12-SDAG-NEXT: v_dual_mov_b32 v14, s5 :: v_dual_mov_b32 v15, s4
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s39, s39
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s11, s11
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s38, s38
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s10, s10
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v5, s37, s37
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v6, s9, s9
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v7, s43, s43
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s15, s15
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v9, s42, s42
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v10, s14, s14
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v11, s41, s41
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v12, s13, s13
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v13, s47, s47
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v14, s19, s19
+; GFX12-SDAG-NEXT: v_dual_min_num_f32 v3, v1, v0 :: v_dual_min_num_f32 v2, v4, v2
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v7, v8, v7
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s46, s46
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s18, s18
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v1, v6, v5
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v6, v10, v9
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s45, s45
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v9, s17, s17
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v5, v12, v11
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v18, s40, s40
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v19, s12, s12
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v10, v4, v0
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v9, v9, v8
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v4, s51, s51
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v8, s23, s23
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v12, s50, s50
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v20, s49, s49
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v21, s21, s21
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v22, s48, s48
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v23, s20, s20
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v11, v14, v13
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v13, s22, s22
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s44, s44
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v24, s16, s16
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v16, s36, s36
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v17, s8, s8
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v15, v8, v4
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v14, v13, v12
+; GFX12-SDAG-NEXT: v_dual_min_num_f32 v13, v21, v20 :: v_dual_min_num_f32 v12, v23, v22
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v8, v24, v0
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v4, v19, v18
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v0, v17, v16
; GFX12-SDAG-NEXT: s_clause 0x3
-; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null offset:48
-; GFX12-SDAG-NEXT: buffer_store_b128 v[4:7], off, s[0:3], null offset:32
-; GFX12-SDAG-NEXT: buffer_store_b128 v[8:11], off, s[0:3], null offset:16
-; GFX12-SDAG-NEXT: buffer_store_b128 v[12:15], off, s[0:3], null
+; GFX12-SDAG-NEXT: buffer_store_b128 v[12:15], off, s[0:3], null offset:48
+; GFX12-SDAG-NEXT: buffer_store_b128 v[8:11], off, s[0:3], null offset:32
+; GFX12-SDAG-NEXT: buffer_store_b128 v[4:7], off, s[0:3], null offset:16
+; GFX12-SDAG-NEXT: buffer_store_b128 v[0:3], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: test_fmin_v16f32:
@@ -1708,6 +1745,10 @@ define <3 x float> @test_func_fmin_v3f32(<3 x float> %a, <3 x float> %b) nounwin
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v3, v3, v3 :: v_dual_max_num_f32 v0, v0, v0
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v4, v4, v4 :: v_dual_max_num_f32 v1, v1, v1
+; GFX12-SDAG-NEXT: v_dual_max_num_f32 v5, v5, v5 :: v_dual_max_num_f32 v2, v2, v2
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_dual_min_num_f32 v0, v0, v3 :: v_dual_min_num_f32 v1, v1, v4
; GFX12-SDAG-NEXT: v_min_num_f32_e32 v2, v2, v5
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -1792,11 +1833,12 @@ define amdgpu_kernel void @test_fmin_f16_v_ieee_on(ptr addrspace(1) %out, half %
; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshr_b32 s3, s2, 16
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: s_min_num_f16 s2, s2, s3
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.l, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.h, s3, s3
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_min_num_f16_e32 v0.l, v0.l, v0.h
; GFX12-SDAG-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -1910,11 +1952,12 @@ define amdgpu_kernel void @test_fmin_f16_s_ieee_on(ptr addrspace(1) %out, half i
; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: s_lshr_b32 s3, s2, 16
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: s_min_num_f16 s2, s2, s3
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.l, s2, s2
+; GFX12-SDAG-NEXT: v_max_num_f16_e64 v0.h, s3, s3
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_min_num_f16_e32 v0.l, v0.l, v0.h
; GFX12-SDAG-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -1960,14 +2003,14 @@ define amdgpu_ps half @test_fmin_f16_s_ieee_off(half inreg %a, half inreg %b) #0
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_min_num_f16 s0, s0, s1
; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: test_fmin_f16_s_ieee_off:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_min_num_f16 s0, s0, s1
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call half @llvm.minnum.f16(half %a, half %b)
ret half %val
@@ -2029,11 +2072,12 @@ define amdgpu_kernel void @test_fmin_f32_s_ieee_on(ptr addrspace(1) %out, float
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: s_min_num_f32 s2, s2, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s3, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s2, s2
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_min_num_f32_e32 v0, v1, v0
; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -2157,9 +2201,12 @@ define amdgpu_kernel void @test_fmin_v2f16_v_ieee_on(ptr addrspace(1) %out, <2 x
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_pk_min_num_f16 v0, s2, s3
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v0, s3, s3
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v1, s2, s2
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_pk_min_num_f16 v0, v1, v0
; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -2284,13 +2331,14 @@ define amdgpu_kernel void @test_fmin_v2f16_s_ieee_on(ptr addrspace(1) %out, <2 x
; GFX12-SDAG-LABEL: test_fmin_v2f16_s_ieee_on:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-SDAG-NEXT: s_mov_b32 s6, -1
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_pk_min_num_f16 v0, s2, s3
-; GFX12-SDAG-NEXT: s_mov_b32 s4, s0
-; GFX12-SDAG-NEXT: s_mov_b32 s5, s1
-; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[4:7], null
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v0, s3, s3
+; GFX12-SDAG-NEXT: v_pk_max_num_f16 v1, s2, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_pk_min_num_f16 v0, v1, v0
+; GFX12-SDAG-NEXT: buffer_store_b32 v0, off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: test_fmin_v2f16_s_ieee_on:
@@ -2410,12 +2458,15 @@ define amdgpu_kernel void @test_fmin_f64_v_ieee_on(ptr addrspace(1) %out, double
; GFX12-SDAG-LABEL: test_fmin_f64_v_ieee_on:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_min_num_f64_e64 v[0:1], s[2:3], s[4:5]
+; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[0:1], s[6:7], s[6:7]
+; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[2:3], s[2:3], s[2:3]
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_min_num_f64_e32 v[0:1], v[2:3], v[0:1]
; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
@@ -2520,12 +2571,15 @@ define amdgpu_kernel void @test_fmin_f64_s_ieee_on(ptr addrspace(1) %out, double
; GFX12-SDAG-LABEL: test_fmin_f64_s_ieee_on:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
+; GFX12-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_min_num_f64_e64 v[0:1], s[2:3], s[4:5]
+; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[0:1], s[6:7], s[6:7]
+; GFX12-SDAG-NEXT: v_max_num_f64_e64 v[2:3], s[2:3], s[2:3]
; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_min_num_f64_e32 v[0:1], v[2:3], v[0:1]
; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
index dbf673fa8cfee..fb4bb5548fe22 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-to-ldexp.ll
@@ -6088,9 +6088,9 @@ define half @v_contract_mul_add_f16_select_64_1(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_64_1:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x3c00
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x5400, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x3c00
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x5400, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6168,9 +6168,9 @@ define half @v_contract_mul_add_f16_select_1_64(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_1_64:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x5400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x3c00, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x5400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x3c00, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6248,9 +6248,9 @@ define half @v_contract_mul_add_f16_select_n64_n1(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_n64_n1:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xbc00
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0xd400, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xbc00
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xd400, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6328,9 +6328,9 @@ define half @v_contract_mul_add_f16_select_n1_n64(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_n1_n64:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0xd400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0xbc00, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xd400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xbc00, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6415,9 +6415,9 @@ define half @v_contract_mul_add_f16_select_128_64(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_128_64:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x5400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x5800, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x5400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x5800, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6501,9 +6501,9 @@ define half @v_contract_mul_add_f16_select_128_4(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_128_4:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x5800, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x4400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x5800, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6588,9 +6588,9 @@ define half @v_contract_mul_add_f16_select_2_4(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_2_4:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x4400
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x4000, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x4400
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x4000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6674,9 +6674,9 @@ define half @v_contract_mul_add_f16_select_4_128(i32 %arg, half %x, half %y) {
; GFX11-SDAG-TRUE16-LABEL: v_contract_mul_add_f16_select_4_128:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, 0x5800
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, 0x4400, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x5800
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x4400, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_fma_f16 v0.l, v1.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
index 87c0367e79ce2..7c832a71993e1 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
@@ -245,12 +245,15 @@ define <2 x i16> @fneg_xor_select_v2i16(<2 x i1> %cond, <2 x i16> %arg0, <2 x i1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v3.h, v2.h, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v1.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v2.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -763,24 +766,29 @@ define <2 x half> @select_fneg_select_v2f16(<2 x i1> %cond0, <2 x i1> %cond1, <2
; GFX11-TRUE16-LABEL: select_fneg_select_v2f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX11-TRUE16-NEXT: v_xor_b32_e32 v1, 0x80008000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v4.l, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v5.h, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v2, 0x80008000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v2.h, s0
+; GFX11-TRUE16-NEXT: v_xor_b32_e32 v3, 0x80008000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v2.l, vcc_lo
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: select_fneg_select_v2f16:
@@ -870,24 +878,29 @@ define <2 x i16> @select_fneg_xor_select_v2i16(<2 x i1> %cond0, <2 x i1> %cond1,
; GFX11-TRUE16-LABEL: select_fneg_xor_select_v2i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX11-TRUE16-NEXT: v_xor_b32_e32 v1, 0x80008000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v1.h, v5.h, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 1, v0.h
-; GFX11-TRUE16-NEXT: v_xor_b32_e32 v2, 0x80008000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v2.h, s0
+; GFX11-TRUE16-NEXT: v_xor_b32_e32 v3, 0x80008000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.h, v2.l, vcc_lo
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: select_fneg_xor_select_v2i16:
@@ -1046,8 +1059,9 @@ define float @cospiD_pattern0_half(i16 %arg, float %arg1, float %arg2) {
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, v1, v2, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x8000, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: cospiD_pattern0_half:
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 64a563b2ddfb1..35bcf309f8ca0 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -1857,7 +1857,8 @@ define <2 x i1> @test_s_unsigned_v2f64_v2i1(<2 x double> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s0, s0, 1
; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i1> @llvm.fptoui.sat.v2f64.v2i1(<2 x double> %f)
ret <2 x i1> %x
@@ -1899,31 +1900,59 @@ define <2 x i8> @test_s_unsigned_v2f64_v2i8(<2 x double> inreg %f) {
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_s_unsigned_v2f64_v2i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
-; GFX11-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
-; GFX11-NEXT: v_min_u32_e32 v1, 0xff, v0
-; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 8, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX11-FAKE16-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v1, 0xff, v0
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-ISEL-LABEL: test_s_unsigned_v2f64_v2i8:
-; GFX12-ISEL: ; %bb.0:
-; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
-; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
-; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
-; GFX12-ISEL-NEXT: v_min_u32_e32 v1, 0xff, v0
-; GFX12-ISEL-NEXT: v_min_u32_e32 v0, 0xff, v2
-; GFX12-ISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v1
-; GFX12-ISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX11-TRUE16-NEXT: v_cvt_u32_f64_e32 v1, s[0:1]
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v0
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX12-FAKE16-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
+; GFX12-FAKE16-NEXT: v_min_u32_e32 v1, 0xff, v0
+; GFX12-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX12-TRUE16-NEXT: v_cvt_u32_f64_e32 v1, s[0:1]
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v0
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v2f64_v2i8:
; GFX12-GI: ; %bb.0:
@@ -1939,7 +1968,8 @@ define <2 x i8> @test_s_unsigned_v2f64_v2i8(<2 x double> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i8> @llvm.fptoui.sat.v2f64.v2i8(<2 x double> %f)
ret <2 x i8> %x
@@ -2366,13 +2396,15 @@ define <4 x i1> @test_unsigned_v4f16_v4i1(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v4f16_v4i1:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: v_min_u16 v0.l, v0.l, 1
-; GFX11-TRUE16-NEXT: v_min_u16 v1.l, v0.h, 1
-; GFX11-TRUE16-NEXT: v_min_u16 v2.l, v2.l, 1
+; GFX11-TRUE16-NEXT: v_min_u16 v2.l, v0.h, 1
+; GFX11-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 1
; GFX11-TRUE16-NEXT: v_min_u16 v3.l, v1.h, 1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2402,13 +2434,15 @@ define <4 x i1> @test_unsigned_v4f16_v4i1(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v3.l
; GFX12-TRUE16-NEXT: v_min_u16 v0.l, v0.l, 1
-; GFX12-TRUE16-NEXT: v_min_u16 v1.l, v0.h, 1
-; GFX12-TRUE16-NEXT: v_min_u16 v2.l, v2.l, 1
+; GFX12-TRUE16-NEXT: v_min_u16 v2.l, v0.h, 1
+; GFX12-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 1
; GFX12-TRUE16-NEXT: v_min_u16 v3.l, v1.h, 1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2526,22 +2560,23 @@ define <4 x i8> @test_unsigned_v4f16_v4i8(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v4f16_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.h
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v2.l
; GFX11-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2580,22 +2615,23 @@ define <4 x i8> @test_unsigned_v4f16_v4i8(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v1.l
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
-; GFX12-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
+; GFX12-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.h
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v2.l
; GFX12-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v0.h
; GFX12-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX12-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2698,10 +2734,12 @@ define <4 x i16> @test_unsigned_v4f16_v4i16(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v4f16_v4i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v4f16_v4i16:
@@ -2728,10 +2766,12 @@ define <4 x i16> @test_unsigned_v4f16_v4i16(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v4f16_v4i16:
@@ -2812,14 +2852,16 @@ define <4 x i32> @test_unsigned_v4f16_v4i32_duplicate(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v4f16_v4i32_duplicate:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v3
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v4
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v1
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v4
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v4f16_v4i32_duplicate:
@@ -2848,14 +2890,16 @@ define <4 x i32> @test_unsigned_v4f16_v4i32_duplicate(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v3
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v4
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v1
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v4
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v4f16_v4i32_duplicate:
@@ -2956,18 +3000,20 @@ define <4 x i64> @test_unsigned_v4f16_v4i64(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v4f16_v4i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v5
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v4f16_v4i64:
@@ -3000,18 +3046,20 @@ define <4 x i64> @test_unsigned_v4f16_v4i64(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.h
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v5
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v1
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v3
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v4f16_v4i64:
@@ -3123,20 +3171,22 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX11-TRUE16-LABEL: test_s_unsigned_v4f16_v4i1:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 1, v2
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 1, v1
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 1, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 1, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v4f16_v4i1:
@@ -3170,21 +3220,24 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
+; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 1, v2
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 1, v1
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 1, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 1, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v4f16_v4i1:
@@ -3215,8 +3268,10 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s1, s1, 1
; GFX12-GI-NEXT: s_min_u32 s3, s3, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptoui.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3342,17 +3397,19 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v2
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v1, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v4f16_v4i8:
@@ -3408,17 +3465,19 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v2
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v1, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v2, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v4f16_v4i8:
@@ -3449,8 +3508,10 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s1, s1, 0xff
; GFX12-GI-NEXT: s_min_u32 s3, s3, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptoui.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -3896,19 +3957,24 @@ define <8 x i1> @test_unsigned_v8f16_v8i1(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v8f16_v8i1:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
-; GFX11-TRUE16-NEXT: v_min_u16 v0.l, v0.l, 1
-; GFX11-TRUE16-NEXT: v_min_u16 v2.l, v1.l, 1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_min_u16 v0.l, v2.l, 1
+; GFX11-TRUE16-NEXT: v_min_u16 v2.l, v2.h, 1
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v6.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v7.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v8.l
; GFX11-TRUE16-NEXT: v_min_u16 v4.l, v4.l, 1
-; GFX11-TRUE16-NEXT: v_min_u16 v6.l, v3.l, 1
-; GFX11-TRUE16-NEXT: v_min_u16 v1.l, v0.h, 1
+; GFX11-TRUE16-NEXT: v_min_u16 v6.l, v0.h, 1
+; GFX11-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 1
; GFX11-TRUE16-NEXT: v_min_u16 v3.l, v1.h, 1
; GFX11-TRUE16-NEXT: v_min_u16 v5.l, v2.h, 1
; GFX11-TRUE16-NEXT: v_min_u16 v7.l, v3.h, 1
@@ -3951,19 +4017,24 @@ define <8 x i1> @test_unsigned_v8f16_v8i1(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
-; GFX12-TRUE16-NEXT: v_min_u16 v0.l, v0.l, 1
-; GFX12-TRUE16-NEXT: v_min_u16 v2.l, v1.l, 1
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_min_u16 v0.l, v2.l, 1
+; GFX12-TRUE16-NEXT: v_min_u16 v2.l, v2.h, 1
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v6.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v7.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v5.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v8.l
; GFX12-TRUE16-NEXT: v_min_u16 v4.l, v4.l, 1
-; GFX12-TRUE16-NEXT: v_min_u16 v6.l, v3.l, 1
-; GFX12-TRUE16-NEXT: v_min_u16 v1.l, v0.h, 1
+; GFX12-TRUE16-NEXT: v_min_u16 v6.l, v0.h, 1
+; GFX12-TRUE16-NEXT: v_min_u16 v1.l, v1.l, 1
; GFX12-TRUE16-NEXT: v_min_u16 v3.l, v1.h, 1
; GFX12-TRUE16-NEXT: v_min_u16 v5.l, v2.h, 1
; GFX12-TRUE16-NEXT: v_min_u16 v7.l, v3.h, 1
@@ -3976,23 +4047,23 @@ define <8 x i1> @test_unsigned_v8f16_v8i1(<8 x half> %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_mov_b32_e32 v4, v2
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.l, v1.l
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, v1.l
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, v4.h
; GFX12-GI-NEXT: v_min_u16 v1.l, v0.h, 1
-; GFX12-GI-NEXT: v_min_u16 v8.l, v4.l, 1
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v2.l
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, v2.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v4.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
; GFX12-GI-NEXT: v_min_u16 v0.l, v0.l, 1
+; GFX12-GI-NEXT: v_min_u16 v2.l, v2.l, 1
; GFX12-GI-NEXT: v_min_u16 v3.l, v1.h, 1
; GFX12-GI-NEXT: v_min_u16 v4.l, v0.h, 1
-; GFX12-GI-NEXT: v_min_u16 v5.l, v2.l, 1
-; GFX12-GI-NEXT: v_min_u16 v6.l, v2.h, 1
+; GFX12-GI-NEXT: v_min_u16 v5.l, v2.h, 1
+; GFX12-GI-NEXT: v_min_u16 v6.l, v4.h, 1
; GFX12-GI-NEXT: v_min_u16 v7.l, v3.h, 1
-; GFX12-GI-NEXT: v_mov_b32_e32 v2, v8
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptoui.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -4166,40 +4237,44 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v8f16_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
-; GFX11-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v6.l
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v0.l
+; GFX11-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v7.l
+; GFX11-TRUE16-NEXT: v_min_u16 v2.l, 0xff, v2.l
+; GFX11-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v4.l
+; GFX11-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX11-TRUE16-NEXT: v_min_u16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_min_u16 v4.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_min_u16 v2.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.h
-; GFX11-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.l, v3.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
-; GFX11-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v8f16_v8i8:
@@ -4257,40 +4332,44 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
-; GFX12-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v5.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v6.l
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v0.l
+; GFX12-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v1.h
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v7.l
+; GFX12-TRUE16-NEXT: v_min_u16 v2.l, 0xff, v2.l
+; GFX12-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v4.l
+; GFX12-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
; GFX12-TRUE16-NEXT: v_min_u16 v3.l, 0xff, v3.l
-; GFX12-TRUE16-NEXT: v_min_u16 v4.l, 0xff, v2.l
-; GFX12-TRUE16-NEXT: v_min_u16 v2.l, 0xff, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v3.h
-; GFX12-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v2.l
-; GFX12-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v0.h
-; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_or_b16 v6.l, v4.l, v3.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
-; GFX12-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX12-TRUE16-NEXT: v_min_u16 v3.h, 0xff, v3.h
+; GFX12-TRUE16-NEXT: v_or_b16 v4.h, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v1.h
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v2.l
+; GFX12-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v2.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: v_or_b16 v4.l, v3.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v8f16_v8i8:
@@ -4300,23 +4379,23 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_mov_b32_e32 v4, v2
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.l, v1.l
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, v1.l
; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, v4.h
; GFX12-GI-NEXT: v_min_u16 v1.l, 0xff, v0.h
-; GFX12-GI-NEXT: v_min_u16 v8.l, 0xff, v4.l
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v2.l
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.l, v2.h
-; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v2.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v0.h, v4.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v4.h, v3.l
+; GFX12-GI-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
; GFX12-GI-NEXT: v_min_u16 v0.l, 0xff, v0.l
+; GFX12-GI-NEXT: v_min_u16 v2.l, 0xff, v2.l
; GFX12-GI-NEXT: v_min_u16 v3.l, 0xff, v1.h
; GFX12-GI-NEXT: v_min_u16 v4.l, 0xff, v0.h
-; GFX12-GI-NEXT: v_min_u16 v5.l, 0xff, v2.l
-; GFX12-GI-NEXT: v_min_u16 v6.l, 0xff, v2.h
+; GFX12-GI-NEXT: v_min_u16 v5.l, 0xff, v2.h
+; GFX12-GI-NEXT: v_min_u16 v6.l, 0xff, v4.h
; GFX12-GI-NEXT: v_min_u16 v7.l, 0xff, v3.h
-; GFX12-GI-NEXT: v_mov_b32_e32 v2, v8
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptoui.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -4455,14 +4534,18 @@ define <8 x i16> @test_unsigned_v8f16_v8i16(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v8f16_v8i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v6.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v8f16_v8i16:
@@ -4497,14 +4580,18 @@ define <8 x i16> @test_unsigned_v8f16_v8i16(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v3.h
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.h, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.h, v5.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.h, v6.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.h, v7.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v8f16_v8i16:
@@ -4635,24 +4722,28 @@ define <8 x i32> @test_unsigned_v8f16_v8i32_duplicate(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_unsigned_v8f16_v8i32_duplicate:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v9, v4
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v8, v5
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v3.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v6
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v7
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v0
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v5
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v5, v2
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v7, v10
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v9
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v6.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v7.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v8, v4
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v9, v5
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v5, v6
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v7, v7
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v1
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v9
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_unsigned_v8f16_v8i32_duplicate:
@@ -4693,24 +4784,28 @@ define <8 x i32> @test_unsigned_v8f16_v8i32_duplicate(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v9, v4
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v8, v5
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v3.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v6
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v7
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v0
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v5
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v5, v2
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v7, v10
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v9
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v5.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v6.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v7.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v8, v4
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v9, v5
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v5, v6
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v7, v7
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v1
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v8
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v9
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_unsigned_v8f16_v8i32_duplicate:
@@ -4877,22 +4972,26 @@ define <8 x i64> @test_unsigned_v8f16_v8i64(<8 x half> %f) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v4
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v8, v6
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v7.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v9, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v8, v7
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.h
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v6
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v12, v9
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v0
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v12, v3
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v6
; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v1
; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v10, v7
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v14, v3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, 0
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v14, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, 0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, 0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
@@ -4947,22 +5046,26 @@ define <8 x i64> @test_unsigned_v8f16_v8i64(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v4
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v8, v6
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v7.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v9, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v5, v4
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v8, v7
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.h
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v6
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v12, v9
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v0
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v5.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v12, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v6
; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v6, v1
; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v10, v7
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v14, v3
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, 0
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v14, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, 0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, 0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, 0
@@ -5159,9 +5262,9 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s5
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s2
@@ -5171,16 +5274,17 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: v_min_u32_e32 v5, 1, v1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v8, 1, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v6, 1, v4
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 1, v8
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 1, v6
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v8f16_v8i1:
@@ -5238,9 +5342,9 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
-; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s2
@@ -5251,16 +5355,17 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v8, 1, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v6, 1, v4
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 1, v8
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 1, v6
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v8f16_v8i1:
@@ -5313,11 +5418,15 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s7, s7, 1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GI-NEXT: s_min_u32 s6, s6, 1
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s5
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v4.l, s4
+; GFX12-GI-NEXT: v_mov_b16_e32 v5.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-GI-NEXT: v_mov_b16_e32 v6.l, s6
+; GFX12-GI-NEXT: v_mov_b16_e32 v7.l, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptoui.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5523,51 +5632,56 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s4
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v5.l, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v7, 0xff, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 8, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v1, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v7, v8
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v3, v11
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v1, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[9:10], 24, v[7:8]
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v3
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s0
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v2, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v3, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v8f16_v8i8:
@@ -5635,54 +5749,58 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
+; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s4
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
-; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v5.l, s1
+; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v7, 0xff, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 8, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v1, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v6
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v7, v8
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, v3, v11
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v1, v7
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[9:10], 24, v[7:8]
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, v1, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v5, v3
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s0
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v6
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v2, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, v3, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v2, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v8
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v9
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v10.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v8f16_v8i8:
@@ -5735,11 +5853,15 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s7, s7, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GI-NEXT: s_min_u32 s6, s6, 0xff
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s5
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v4.l, s4
+; GFX12-GI-NEXT: v_mov_b16_e32 v5.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-GI-NEXT: v_mov_b16_e32 v6.l, s6
+; GFX12-GI-NEXT: v_mov_b16_e32 v7.l, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptoui.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -6390,52 +6512,23 @@ define <4 x i16> @test_s_unsigned_v4f32_v4i16(<4 x float> inreg %f) {
; GFX9-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: test_s_unsigned_v4f32_v4i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
-; GFX11-FAKE16-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: test_s_unsigned_v4f32_v4i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_pk_u16_f32 v0, s0, s0
-; GFX11-TRUE16-NEXT: v_cvt_pk_u16_f32 v1, s2, s2
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: test_s_unsigned_v4f32_v4i16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
-; GFX12-FAKE16-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-TRUE16-LABEL: test_s_unsigned_v4f32_v4i16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_pk_u16_f32 v0, s0, s0
-; GFX12-TRUE16-NEXT: v_cvt_pk_u16_f32 v1, s2, s2
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_s_unsigned_v4f32_v4i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
+; GFX11-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-GI-LABEL: test_s_unsigned_v4f32_v4i16:
-; GFX12-GI: ; %bb.0:
-; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GI-NEXT: s_wait_expcnt 0x0
-; GFX12-GI-NEXT: s_wait_samplecnt 0x0
-; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
-; GFX12-GI-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
-; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: test_s_unsigned_v4f32_v4i16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
+; GFX12-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i16> @llvm.fptoui.sat.v4f32.v4i16(<4 x float> %f)
ret <4 x i16> %x
}
@@ -6525,21 +6618,20 @@ define <4 x i8> @test_unsigned_v4f32_v4i8(<4 x float> %f) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v2
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v1
; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v4
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6577,21 +6669,20 @@ define <4 x i8> @test_unsigned_v4f32_v4i8(<4 x float> %f) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v2
-; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v4, v1
; GFX12-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v4
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -6679,26 +6770,49 @@ define <4 x i8> @test_s_unsigned_v4f32_v4i8(<4 x float> inreg %f) {
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_s_unsigned_v4f32_v4i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, s3
-; GFX11-NEXT: v_cvt_u32_f32_e32 v1, s2
-; GFX11-NEXT: v_cvt_u32_f32_e32 v2, s1
-; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-NEXT: v_min_u32_e32 v1, 0xff, v1
-; GFX11-NEXT: v_min_u32_e32 v3, 0xff, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-NEXT: v_or_b32_e32 v2, v1, v0
-; GFX11-NEXT: v_cvt_u32_f32_e32 v0, s0
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-LABEL: test_s_unsigned_v4f32_v4i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v0, s3
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v1, s2
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v2, s1
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v3, 0xff, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v1, v0
+; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v0, s0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v1, v3
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-TRUE16-LABEL: test_s_unsigned_v4f32_v4i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, s3
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, s2
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, s1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v4f32_v4i8:
; GFX12-FAKE16: ; %bb.0:
@@ -6760,14 +6874,14 @@ define <4 x i8> @test_s_unsigned_v4f32_v4i8(<4 x float> inreg %f) {
; GFX12-TRUE16-NEXT: s_or_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s1, s1, s3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 24
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshr_b32 s0, s1, 8
-; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s3, 24
+; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, s1
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, s1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v4f32_v4i8:
@@ -6787,8 +6901,10 @@ define <4 x i8> @test_s_unsigned_v4f32_v4i8(<4 x float> inreg %f) {
; GFX12-GI-NEXT: s_min_u32 s2, s2, 0xff
; GFX12-GI-NEXT: s_min_u32 s3, s3, 0xff
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptoui.sat.v4f32.v4i8(<4 x float> %f)
ret <4 x i8> %x
diff --git a/llvm/test/CodeGen/AMDGPU/function-args.ll b/llvm/test/CodeGen/AMDGPU/function-args.ll
index e36a807a7ceed..d17edcb72f992 100644
--- a/llvm/test/CodeGen/AMDGPU/function-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-args.ll
@@ -1347,48 +1347,26 @@ define void @void_func_v16i8(<16 x i8> %arg0) #0 {
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: void_func_v16i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v8, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v11.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v10, v9, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v2, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; GFX11-TRUE16-NEXT: s_mov_b64 s[0:1], 0
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: void_func_v16i8:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v8, v8, v9, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v9, v10, v11, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v6, v2, v3, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], 0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX11-FAKE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: void_func_v16i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v8, v8, v9, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v9, v10, v11, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v6, v2, v3, 0xc0c0004
+; GFX11-NEXT: v_lshl_or_b32 v3, v13, 16, v12
+; GFX11-NEXT: v_lshl_or_b32 v2, v9, 16, v8
+; GFX11-NEXT: v_lshl_or_b32 v1, v5, 16, v4
+; GFX11-NEXT: s_mov_b64 s[0:1], 0
+; GFX11-NEXT: v_lshl_or_b32 v0, v6, 16, v0
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mov_b32 s2, -1
+; GFX11-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
store <16 x i8> %arg0, ptr addrspace(1) null
ret void
}
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index 019e49a823aa2..66a27bf6258c4 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -1113,24 +1113,35 @@ define <16 x i8> @v16i8_func_void() #0 {
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_load_b128 v[16:19], off, s[0:3], 0
+; GFX11-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v16i8_func_void:
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index fd0ddea06a115..14212cb73727b 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -3896,16 +3896,21 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v8i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v8i8 at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b64 v[8:9], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
@@ -4155,45 +4160,67 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8() #0 {
; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 0
; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, 16 :: v_dual_mov_b32 v5, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v32i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v32i8 at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b128 v[32:35], v[0:1], off
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 16 :: v_dual_mov_b32 v1, 0
-; GFX11-TRUE16-NEXT: global_load_b128 v[36:39], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v[4:5], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v39
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v39.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v37.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
@@ -4650,10 +4677,9 @@ define amdgpu_gfx void @test_call_external_void_func_v2i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-TRUE16-NEXT: global_store_b16 v[40:41], v0, off
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
@@ -4844,93 +4870,49 @@ define amdgpu_gfx void @test_call_external_void_func_v3i8_ret() #0 {
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: test_call_external_void_func_v3i8_ret:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
-; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
-; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s0, 2
-; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 16
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Spill
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 offset:4
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s33
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s30, 0
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s31, 1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[40:41], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, 2 :: v_dual_mov_b32 v4, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: global_store_b8 v[3:4], v2, off
-; GFX11-TRUE16-NEXT: global_store_b16 v[40:41], v0, off
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
-; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
-; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
-; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: test_call_external_void_func_v3i8_ret:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
-; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
-; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s0, 2
-; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 16
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Spill
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 offset:4
-; GFX11-FAKE16-NEXT: ; meta instruction
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v41, s33
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s30, 0
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s31, 1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[40:41], off
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, 2 :: v_dual_mov_b32 v4, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v42, 0
-; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: global_store_b8 v[3:4], v2, off
-; GFX11-FAKE16-NEXT: global_store_b16 v[40:41], v0, off
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
-; GFX11-FAKE16-NEXT: scratch_load_b32 v41, off, s33
-; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
-; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v42, 2
-; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-FAKE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_call_external_void_func_v3i8_ret:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s0, s33
+; GFX11-NEXT: s_mov_b32 s33, s32
+; GFX11-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: v_writelane_b32 v42, s0, 2
+; GFX11-NEXT: s_add_i32 s32, s32, 16
+; GFX11-NEXT: s_clause 0x1 ; 8-byte Folded Spill
+; GFX11-NEXT: scratch_store_b32 off, v40, s33 offset:4
+; GFX11-NEXT: ; meta instruction
+; GFX11-NEXT: scratch_store_b32 off, v41, s33
+; GFX11-NEXT: v_writelane_b32 v42, s30, 0
+; GFX11-NEXT: v_writelane_b32 v42, s31, 1
+; GFX11-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
+; GFX11-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
+; GFX11-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
+; GFX11-NEXT: global_load_b32 v0, v[40:41], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v3, 2 :: v_dual_mov_b32 v4, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX11-NEXT: v_readlane_b32 s30, v42, 0
+; GFX11-NEXT: v_readlane_b32 s31, v42, 1
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: global_store_b8 v[3:4], v2, off
+; GFX11-NEXT: global_store_b16 v[40:41], v0, off
+; GFX11-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; GFX11-NEXT: scratch_load_b32 v41, off, s33
+; GFX11-NEXT: scratch_load_b32 v40, off, s33 offset:4
+; GFX11-NEXT: s_mov_b32 s32, s33
+; GFX11-NEXT: v_readlane_b32 s0, v42, 2
+; GFX11-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 s33, s0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_v3i8_ret:
; GFX10-SCRATCH: ; %bb.0:
@@ -5070,94 +5052,49 @@ define amdgpu_gfx void @test_call_external_void_func_v4i8_ret() #0 {
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: test_call_external_void_func_v4i8_ret:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
-; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
-; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s0, 2
-; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 16
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Spill
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 offset:4
-; GFX11-TRUE16-NEXT: ; meta instruction
-; GFX11-TRUE16-NEXT: scratch_store_b32 off, v41, s33
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s30, 0
-; GFX11-TRUE16-NEXT: v_writelane_b32 v42, s31, 1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v4i8_ret at abs32@hi
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v4i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[40:41], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
-; GFX11-TRUE16-NEXT: global_store_b32 v[40:41], v0, off
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
-; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
-; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
-; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: test_call_external_void_func_v4i8_ret:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
-; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
-; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s0, 2
-; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 16
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Spill
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 offset:4
-; GFX11-FAKE16-NEXT: ; meta instruction
-; GFX11-FAKE16-NEXT: scratch_store_b32 off, v41, s33
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s30, 0
-; GFX11-FAKE16-NEXT: v_writelane_b32 v42, s31, 1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
-; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_v4i8_ret at abs32@hi
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_v4i8_ret at abs32@lo
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[40:41], off
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v42, 0
-; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
-; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v42, 2
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-FAKE16-NEXT: global_store_b32 v[40:41], v0, off
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
-; GFX11-FAKE16-NEXT: scratch_load_b32 v41, off, s33
-; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-FAKE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
-; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_call_external_void_func_v4i8_ret:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s0, s33
+; GFX11-NEXT: s_mov_b32 s33, s32
+; GFX11-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-NEXT: scratch_store_b32 off, v42, s33 offset:8 ; 4-byte Folded Spill
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: v_writelane_b32 v42, s0, 2
+; GFX11-NEXT: s_add_i32 s32, s32, 16
+; GFX11-NEXT: s_clause 0x1 ; 8-byte Folded Spill
+; GFX11-NEXT: scratch_store_b32 off, v40, s33 offset:4
+; GFX11-NEXT: ; meta instruction
+; GFX11-NEXT: scratch_store_b32 off, v41, s33
+; GFX11-NEXT: v_writelane_b32 v42, s30, 0
+; GFX11-NEXT: v_writelane_b32 v42, s31, 1
+; GFX11-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
+; GFX11-NEXT: s_mov_b32 s1, external_void_func_v4i8_ret at abs32@hi
+; GFX11-NEXT: s_mov_b32 s0, external_void_func_v4i8_ret at abs32@lo
+; GFX11-NEXT: global_load_b32 v0, v[40:41], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
+; GFX11-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
+; GFX11-NEXT: v_readlane_b32 s30, v42, 0
+; GFX11-NEXT: v_readlane_b32 s31, v42, 1
+; GFX11-NEXT: s_mov_b32 s32, s33
+; GFX11-NEXT: v_readlane_b32 s0, v42, 2
+; GFX11-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-NEXT: global_store_b32 v[40:41], v0, off
+; GFX11-NEXT: s_clause 0x1 ; 8-byte Folded Reload
+; GFX11-NEXT: scratch_load_b32 v41, off, s33
+; GFX11-NEXT: scratch_load_b32 v40, off, s33 offset:4
+; GFX11-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
+; GFX11-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-NEXT: s_mov_b32 s33, s0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_v4i8_ret:
; GFX10-SCRATCH: ; %bb.0:
@@ -5336,20 +5273,20 @@ define amdgpu_gfx void @test_call_external_void_func_v5i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v0, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 4 :: v_dual_mov_b32 v1, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v2, v2, 16, v5
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: global_store_b8 v[0:1], v4, off
; GFX11-TRUE16-NEXT: global_store_b32 v[40:41], v2, off
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
@@ -5584,33 +5521,37 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v8i8_ret at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v8i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b64 v[8:9], v[40:41], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[40:41], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
; GFX11-TRUE16-NEXT: global_store_b64 v[40:41], v[0:1], off
; GFX11-TRUE16-NEXT: s_clause 0x1 ; 8-byte Folded Reload
; GFX11-TRUE16-NEXT: scratch_load_b32 v41, off, s33
; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 offset:4
-; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v42, 1
-; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
-; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v42, 2
; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
; GFX11-TRUE16-NEXT: scratch_load_b32 v42, off, s33 offset:8 ; 4-byte Folded Reload
; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
@@ -5967,42 +5908,64 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v42, 16 :: v_dual_mov_b32 v43, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b128 v[32:35], v[40:41], off
-; GFX11-TRUE16-NEXT: global_load_b128 v[36:39], v[42:43], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v[40:41], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v[42:43], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v39
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v39.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v21.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v23.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v27.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v29.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v30.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v37.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
@@ -9693,26 +9656,37 @@ define amdgpu_gfx void @test_call_external_void_func_v16i8() #0 {
; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v0, s[0:1]
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v0, s[0:1]
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v16i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v16i8 at abs32@lo
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index d3165eba9c26b..ee52b70b953e2 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13717,58 +13717,114 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -13813,54 +13869,106 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14154,73 +14262,131 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB55_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v3, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX942-NEXT: s_mov_b32 s0, 0xffff
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: s_mov_b32 s0, 0xffff
; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
; GFX942-NEXT: v_not_b32_e32 v5, v5
; GFX942-NEXT: s_mov_b64 s[0:1], 0
@@ -14255,56 +14421,110 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB55_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14607,61 +14827,119 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB56_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14709,56 +14987,110 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB56_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -15058,56 +15390,111 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB57_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -15151,71 +15538,122 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB57_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v3, v0
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX10-NEXT: s_mov_b32 s4, 0
-; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX10-NEXT: v_not_b32_e32 v6, v3
-; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v3, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX10-NEXT: s_mov_b32 s4, 0
+; GFX10-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX10-NEXT: global_load_dword v4, v[0:1], off
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v6, v3
+; GFX10-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_lshrrev_b32_sdwa v3, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX10-NEXT: v_bfe_u32 v7, v3, 16, 1
; GFX10-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX10-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX10-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
@@ -15441,90 +15879,47 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
}
define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v5, v5
-; GFX1250-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-FAKE16-NEXT: .LBB58_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB58_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v6, v4
+; GFX1250-NEXT: .LBB58_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v4, v4, v2
+; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, v3, v4
+; GFX1250-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v4
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB58_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
@@ -15924,92 +16319,48 @@ define void @global_agent_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_
define void @global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
;
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v5, v5
-; GFX1250-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-FAKE16-NEXT: .LBB59_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB59_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_mov_b64 s[0:1], 0xfffffffffffff800
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], s[0:1], v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: v_not_b32_e32 v6, v4
+; GFX1250-NEXT: .LBB59_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v4, v4, v2
+; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, v3, v4
+; GFX1250-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v4
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB59_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
@@ -16479,32 +16830,33 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16512,7 +16864,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -16597,40 +16949,41 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -16882,66 +17235,35 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
define void @global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
;
-; GFX1250-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v3, v4
-; GFX1250-TRUE16-NEXT: v_bfi_b32 v2, 0xffff0000, v3, v2
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off offset:2046
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: .LBB61_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v3, v5, v2
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v3
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB61_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off offset:2046
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: .LBB61_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_pk_add_bf16 v3, v5, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_bfi_b32 v4, 0xffff0000, v5, v3
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_DEV
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v3
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB61_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
@@ -17305,62 +17627,121 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB62_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -17407,56 +17788,110 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB62_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -17718,90 +18153,47 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
define void @global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
;
-; GFX1250-TRUE16-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
-; GFX1250-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_not_b32_e32 v5, v5
-; GFX1250-TRUE16-NEXT: .LBB63_1: ; %atomicrmw.start
-; GFX1250-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_pk_add_bf16 v2, v2, v6
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX1250-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX1250-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_cbranch_execnz .LBB63_1
-; GFX1250-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-FAKE16-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
-; GFX1250-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v3, 3, v4
-; GFX1250-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_not_b32_e32 v6, v4
-; GFX1250-FAKE16-NEXT: .LBB63_1: ; %atomicrmw.start
-; GFX1250-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v4, v3, v5
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_pk_add_bf16 v4, v4, v2
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_lshlrev_b32_e32 v4, v3, v4
-; GFX1250-FAKE16-NEXT: v_and_or_b32 v4, v5, v6, v4
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX1250-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_cbranch_execnz .LBB63_1
-; GFX1250-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX1250-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_add_nc_u64_e32 v[4:5], 0x7fe, v[0:1]
+; GFX1250-NEXT: s_mov_b32 s0, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_bitop2_b32 v0, -4, v4 bitop3:0x40
+; GFX1250-NEXT: v_and_b32_e32 v3, 3, v4
+; GFX1250-NEXT: global_load_b32 v5, v[0:1], off
+; GFX1250-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX1250-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_not_b32_e32 v6, v4
+; GFX1250-NEXT: .LBB63_1: ; %atomicrmw.start
+; GFX1250-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_lshrrev_b32_e32 v4, v3, v5
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_pk_add_bf16 v4, v4, v2
+; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_lshlrev_b32_e32 v4, v3, v4
+; GFX1250-NEXT: v_and_or_b32 v4, v5, v6, v4
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: global_wb scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_storecnt 0x0
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: global_atomic_cmpswap_b32 v4, v[0:1], v[4:5], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: global_inv scope:SCOPE_SYS
+; GFX1250-NEXT: s_wait_loadcnt 0x0
+; GFX1250-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v5
+; GFX1250-NEXT: v_mov_b32_e32 v5, v4
+; GFX1250-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_cbranch_execnz .LBB63_1
+; GFX1250-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX12-LABEL: global_system_atomic_fadd_noret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX12: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index fe49de1f0d29f..3f674bbabca01 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -3015,13 +3015,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3186,13 +3188,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3358,13 +3362,15 @@ define double @global_agent_atomic_fmax_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3529,18 +3535,21 @@ define void @global_agent_atomic_fmax_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3689,18 +3698,21 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3852,18 +3864,21 @@ define void @global_agent_atomic_fmax_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4016,13 +4031,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -4262,13 +4279,15 @@ define double @global_agent_atomic_fmax_ret_f64__amdgpu_no_fine_grained_memory__
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -4437,8 +4456,9 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4452,11 +4472,12 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4481,8 +4502,9 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4496,11 +4518,12 @@ define half @global_agent_atomic_fmax_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4869,34 +4892,36 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4904,7 +4929,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -4917,13 +4942,15 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -4931,11 +4958,12 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4999,40 +5027,38 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5318,34 +5344,36 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5353,7 +5381,7 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5366,13 +5394,15 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -5380,11 +5410,12 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5449,40 +5480,38 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5769,8 +5798,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5783,10 +5813,12 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5811,8 +5843,9 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5825,10 +5858,12 @@ define void @global_agent_atomic_fmax_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6189,33 +6224,36 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6232,33 +6270,37 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6316,31 +6358,29 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6625,33 +6665,36 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6668,33 +6711,37 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6753,31 +6800,29 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7063,15 +7108,17 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7096,15 +7143,17 @@ define half @global_agent_atomic_fmax_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v4
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7399,13 +7448,16 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7429,21 +7481,24 @@ define void @global_agent_atomic_fmax_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v3, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7722,35 +7777,37 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7758,7 +7815,7 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7771,13 +7828,15 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7785,11 +7844,12 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
@@ -7854,40 +7914,38 @@ define half @global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_system_atomic_fmax_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8176,34 +8234,37 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8220,34 +8281,38 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8305,31 +8370,29 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8614,58 +8677,114 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -8710,54 +8829,106 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9009,85 +9180,143 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v3, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
-; GFX942-NEXT: v_not_b32_e32 v5, v5
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX942-NEXT: s_movk_i32 s2, 0x7fff
-; GFX942-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_max_f32_e32 v2, v2, v6
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: s_mov_b32 s0, 0xffff
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX942-NEXT: s_movk_i32 s2, 0x7fff
+; GFX942-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX942-NEXT: v_add3_u32 v7, v7, v2, s2
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
@@ -9110,56 +9339,110 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9419,61 +9702,119 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9521,56 +9862,110 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9826,60 +10221,115 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
%result = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
- ret bfloat %result
- }
-
-define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+ ret bfloat %result
+ }
+
+define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9923,52 +10373,103 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11021,32 +11522,33 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11054,7 +11556,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11139,40 +11641,41 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11716,90 +12219,149 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_max_f32_e32 v2, v2, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX6-NEXT: v_mov_b32_e32 v6, v3
-; GFX6-NEXT: v_mov_b32_e32 v5, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v3, v5
-; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB43_1
-; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
- ret void
-}
-
-define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX6-NEXT: v_mov_b32_e32 v6, v3
+; GFX6-NEXT: v_mov_b32_e32 v5, v2
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
+; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v3, v5
+; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_cbranch_execnz .LBB43_1
+; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
+ %unused = atomicrmw fmax ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -11846,56 +12408,110 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12573,13 +13189,15 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__amdgpu_no_fine_grained_me
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -12853,13 +13471,15 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -13135,13 +13755,15 @@ define <2 x half> @global_agent_atomic_fmax_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -13424,18 +14046,21 @@ define void @global_agent_atomic_fmax_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13690,18 +14315,21 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13959,18 +14587,21 @@ define void @global_agent_atomic_fmax_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14237,13 +14868,15 @@ define <2 x half> @global_system_atomic_fmax_ret_v2f16__offset12b_pos__amdgpu_no
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_max_num_f16 v3, v3, v2
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -14522,19 +15155,22 @@ define void @global_system_atomic_fmax_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v4
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14808,12 +15444,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -14825,10 +15461,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -14945,18 +15584,19 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -14967,9 +15607,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -14981,6 +15624,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -15315,12 +15959,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15332,10 +15976,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -15452,18 +16099,19 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -15474,9 +16122,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -15488,6 +16139,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -15824,12 +16476,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15841,10 +16493,13 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -15961,18 +16616,19 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -15983,9 +16639,12 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -15997,6 +16656,7 @@ define <2 x bfloat> @global_agent_atomic_fmax_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16339,11 +16999,11 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16353,12 +17013,15 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16473,15 +17136,16 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16490,12 +17154,15 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -16508,6 +17175,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -16825,11 +17493,11 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16839,12 +17507,15 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16959,15 +17630,16 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16976,12 +17648,15 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -16994,6 +17669,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17314,11 +17990,11 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17328,12 +18004,15 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -17448,15 +18127,16 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17465,12 +18145,15 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17483,6 +18166,7 @@ define void @global_agent_atomic_fmax_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17813,12 +18497,12 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -17830,10 +18514,13 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -17952,18 +18639,19 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -17974,9 +18662,12 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17988,6 +18679,7 @@ define <2 x bfloat> @global_system_atomic_fmax_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -18325,11 +19017,11 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -18339,12 +19031,15 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -18461,15 +19156,16 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -18478,12 +19174,15 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -18496,6 +19195,7 @@ define void @global_system_atomic_fmax_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 0095f991e122d..7c09a40e7c578 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -3015,13 +3015,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB18_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3186,13 +3188,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB19_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3358,13 +3362,15 @@ define double @global_agent_atomic_fmin_ret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB20_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3529,18 +3535,21 @@ define void @global_agent_atomic_fmin_noret_f64__amdgpu_no_fine_grained_memory(p
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB21_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3689,18 +3698,21 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_pos__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:2040
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:2040
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB22_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:2040 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -3852,18 +3864,21 @@ define void @global_agent_atomic_fmin_noret_f64__offset12b_neg__amdgpu_no_fine_g
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b64 v[6:7], v[0:1], off offset:-2048
+; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off offset:-2048
+; GFX12-NEXT: v_max_num_f64_e32 v[6:7], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB23_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[4:5], v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f64_e32 v[2:3], v[2:3], v[6:7]
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b64 v[2:3], v[0:1], v[2:5], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[4:5], v[6:7]
-; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
+; GFX12-NEXT: v_cmp_eq_u64_e32 vcc_lo, v[2:3], v[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4016,13 +4031,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_remote_memory(ptr add
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB24_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -4262,13 +4279,15 @@ define double @global_agent_atomic_fmin_ret_f64__amdgpu_no_fine_grained_memory__
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b64 v[4:5], v[0:1], off
+; GFX12-NEXT: v_max_num_f64_e32 v[2:3], v[2:3], v[2:3]
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB25_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_dual_mov_b32 v7, v5 :: v_dual_mov_b32 v6, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[6:7], v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f64_e32 v[4:5], v[6:7], v[6:7]
+; GFX12-NEXT: v_min_num_f64_e32 v[4:5], v[4:5], v[2:3]
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b64 v[4:5], v[0:1], v[4:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -4437,8 +4456,9 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4452,11 +4472,12 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v5.l, v5.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4481,8 +4502,9 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
@@ -4496,11 +4518,12 @@ define half @global_agent_atomic_fmin_ret_f16__amdgpu_no_fine_grained_memory(ptr
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4869,34 +4892,36 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -4904,7 +4929,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -4917,13 +4942,15 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -4931,11 +4958,12 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -4999,40 +5027,38 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB27_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB27_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -5318,34 +5344,36 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -5353,7 +5381,7 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5366,13 +5394,15 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -5380,11 +5410,12 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5449,40 +5480,38 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_gra
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB28_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB28_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_f16__offset12b_neg__amdgpu_no_fine_grained_memory:
@@ -5769,8 +5798,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5783,10 +5813,12 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v3.l, v3.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -5811,8 +5843,9 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
@@ -5825,10 +5858,12 @@ define void @global_agent_atomic_fmin_noret_f16__amdgpu_no_fine_grained_memory(p
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v3, v3
; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -6189,33 +6224,36 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6232,33 +6270,37 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6316,31 +6358,29 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB30_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -6625,33 +6665,36 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff800, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, -1, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6668,33 +6711,37 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0xfffff800, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -6753,31 +6800,29 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b_neg__amdgpu_no_fine_g
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB31_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -7063,15 +7108,17 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7096,15 +7143,17 @@ define half @global_agent_atomic_fmin_ret_f16__offset12b_pos__align4__amdgpu_no_
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v4, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v3, v4, v4
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7399,13 +7448,16 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v4.l, v2.l
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v2.h, v4.l, v4.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v2.h, v2.l
; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
@@ -7429,21 +7481,24 @@ define void @global_agent_atomic_fmin_noret_f16__offset12b__align4_pos__amdgpu_n
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v4, v2, v2
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v4, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v3, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, 0xffff0000, v3, v2
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7722,35 +7777,37 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7758,7 +7815,7 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -7771,13 +7828,15 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7785,11 +7844,12 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v5, v5, v5
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v5, v5, v2
; GFX12-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
@@ -7854,40 +7914,38 @@ define half @global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_gr
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[3:4], off
+; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v6, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v7
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v6, v7, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v6, v[3:4], v[6:7], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v6, v7
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v1, v6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_system_atomic_fmin_ret_f16__offset12b_pos__amdgpu_no_fine_grained_memory:
@@ -8176,34 +8234,37 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, 0x7fe, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v1, vcc_lo
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, -4, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX12-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.l, v2.l, v2.l
; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v2.h, v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v2.h
+; GFX12-TRUE16-NEXT: v_max_num_f16_e32 v0.h, v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_min_num_f16_e32 v0.h, v0.h, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8220,34 +8281,38 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v4, vcc_lo, 0x7fe, v0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v6, v2, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v4, 3, v4
; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v5, v5
; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-FAKE16-NEXT: v_max_num_f16_e32 v2, v2, v2
+; GFX12-FAKE16-NEXT: v_min_num_f16_e32 v2, v2, v6
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v2, v3, v5, v2
; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v2
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -8305,31 +8370,29 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 3, v0
; GFX11-TRUE16-NEXT: global_load_b32 v6, v[3:4], off
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v5, v1, 0xffff
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v0.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, v1, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v7, v0
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.l, v2.l, v2.l
; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v1, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v1, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v5.l, v5.l
+; GFX11-TRUE16-NEXT: v_max_f16_e32 v0.h, v2.l, v2.l
; GFX11-TRUE16-NEXT: v_min_f16_e32 v0.h, v0.h, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v1, v5
+; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v1, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v2, v5
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v7, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[3:4], v[5:6], off glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[3:4], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
@@ -8614,58 +8677,114 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -8710,54 +8829,106 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9009,85 +9180,143 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
-; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
-; GFX942-NEXT: v_mov_b32_e32 v1, v5
-; GFX942-NEXT: global_load_dword v3, v[0:1], off
-; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
-; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
-; GFX942-NEXT: s_mov_b32 s0, 0xffff
-; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
-; GFX942-NEXT: v_not_b32_e32 v5, v5
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX942-NEXT: s_movk_i32 s2, 0x7fff
-; GFX942-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_min_f32_e32 v2, v2, v6
-; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: s_mov_b64 s[0:1], 0x7fe
+; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[0:1]
+; GFX942-NEXT: v_and_b32_e32 v0, -4, v4
+; GFX942-NEXT: v_mov_b32_e32 v1, v5
+; GFX942-NEXT: global_load_dword v3, v[0:1], off
+; GFX942-NEXT: v_and_b32_e32 v4, 3, v4
+; GFX942-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX942-NEXT: s_mov_b32 s0, 0xffff
+; GFX942-NEXT: v_lshlrev_b32_e64 v5, v4, s0
+; GFX942-NEXT: v_not_b32_e32 v5, v5
+; GFX942-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX942-NEXT: s_movk_i32 s2, 0x7fff
+; GFX942-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: v_lshrrev_b32_sdwa v2, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX942-NEXT: s_nop 0
+; GFX942-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX942-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX942-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX942-NEXT: v_add3_u32 v7, v7, v2, s2
; GFX942-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
@@ -9110,56 +9339,110 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9419,61 +9702,119 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9521,56 +9862,110 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9826,60 +10221,115 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX6-NEXT: s_setpc_b64 s[30:31]
%gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 -1024
%result = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, !amdgpu.no.fine.grained.memory !0
- ret bfloat %result
- }
-
-define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+ ret bfloat %result
+ }
+
+define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9923,52 +10373,103 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11021,32 +11522,33 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11054,7 +11556,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11139,40 +11641,41 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11716,90 +12219,149 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
; GFX6-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX6-NEXT: .LBB43_1: ; %atomicrmw.start
; GFX6-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
-; GFX6-NEXT: v_min_f32_e32 v2, v2, v4
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX6-NEXT: v_mov_b32_e32 v6, v3
-; GFX6-NEXT: v_mov_b32_e32 v5, v2
-; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_wbinvl1
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
-; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GFX6-NEXT: v_mov_b32_e32 v3, v5
-; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_cbranch_execnz .LBB43_1
-; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: s_setpc_b64 s[30:31]
- %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
- %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
- ret void
-}
-
-define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_mul_f32_e32 v2, 1.0, v2
+; GFX6-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX6-NEXT: v_mov_b32_e32 v6, v3
+; GFX6-NEXT: v_mov_b32_e32 v5, v2
+; GFX6-NEXT: buffer_atomic_cmpswap v[5:6], v[0:1], s[4:7], 0 addr64 offset:2046 glc
+; GFX6-NEXT: s_waitcnt vmcnt(0)
+; GFX6-NEXT: buffer_wbinvl1
+; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3
+; GFX6-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
+; GFX6-NEXT: v_mov_b32_e32 v3, v5
+; GFX6-NEXT: s_andn2_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_cbranch_execnz .LBB43_1
+; GFX6-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX6-NEXT: s_or_b64 exec, exec, s[8:9]
+; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_setpc_b64 s[30:31]
+ %gep = getelementptr bfloat, ptr addrspace(1) %ptr, i64 1023
+ %unused = atomicrmw fmin ptr addrspace(1) %gep, bfloat %val syncscope("agent") seq_cst, align 4, !amdgpu.no.fine.grained.memory !0
+ ret void
+}
+
+define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
+; GFX12-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -11846,56 +12408,110 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12573,13 +13189,15 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__amdgpu_no_fine_grained_me
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -12853,13 +13471,15 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -13135,13 +13755,15 @@ define <2 x half> @global_agent_atomic_fmin_ret_v2f16__offset12b_neg__amdgpu_no_
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -13424,18 +14046,21 @@ define void @global_agent_atomic_fmin_noret_v2f16__amdgpu_no_fine_grained_memory
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB49_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13690,18 +14315,21 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fine
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB50_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -13959,18 +14587,21 @@ define void @global_agent_atomic_fmin_noret_v2f16__offset12b_neg__amdgpu_no_fine
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:-2048
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:-2048
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14237,13 +14868,15 @@ define <2 x half> @global_system_atomic_fmin_ret_v2f16__offset12b_pos__amdgpu_no
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v2, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB52_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_pk_max_num_f16 v3, v4, v4
+; GFX12-NEXT: v_pk_min_num_f16 v3, v3, v2
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -14522,19 +15155,22 @@ define void @global_system_atomic_fmin_noret_v2f16__offset12b_pos__amdgpu_no_fin
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off offset:2044
+; GFX12-NEXT: global_load_b32 v3, v[0:1], off offset:2044
+; GFX12-NEXT: v_pk_max_num_f16 v4, v2, v2
; GFX12-NEXT: s_mov_b32 s0, 0
; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_pk_min_num_f16 v3, v4, v2
+; GFX12-NEXT: v_pk_max_num_f16 v2, v3, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_pk_min_num_f16 v2, v2, v4
; GFX12-NEXT: global_wb scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-NEXT: v_mov_b32_e32 v3, v2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14808,12 +15444,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -14825,10 +15461,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -14945,18 +15584,19 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -14967,9 +15607,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -14981,6 +15624,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__amdgpu_no_fine_grained
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -15315,12 +15959,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15332,10 +15976,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -15452,18 +16099,19 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -15474,9 +16122,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -15488,6 +16139,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -15824,12 +16476,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -15841,10 +16493,13 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -15961,18 +16616,19 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -15983,9 +16639,12 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -15997,6 +16656,7 @@ define <2 x bfloat> @global_agent_atomic_fmin_ret_v2bf16__offset12b_neg__amdgpu_
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16339,11 +16999,11 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16353,12 +17013,15 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16473,15 +17136,16 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16490,12 +17154,15 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -16508,6 +17175,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__amdgpu_no_fine_grained_memor
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -16825,11 +17493,11 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16839,12 +17507,15 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -16959,15 +17630,16 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB58_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -16976,12 +17648,15 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -16994,6 +17669,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB58_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17314,11 +17990,11 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17328,12 +18004,15 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
@@ -17448,15 +18127,16 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB59_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -17465,12 +18145,15 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:-2048 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17483,6 +18166,7 @@ define void @global_agent_atomic_fmin_noret_v2bf16__offset12b_neg__amdgpu_no_fin
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB59_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -17813,12 +18497,12 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v2
; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -17830,10 +18514,13 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -17952,18 +18639,19 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
@@ -17974,9 +18662,12 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v3.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[5:6], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -17988,6 +18679,7 @@ define <2 x bfloat> @global_system_atomic_fmin_ret_v2bf16__offset12b_pos__amdgpu
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -18325,11 +19017,11 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v6, v6, v5
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -18339,12 +19031,15 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -18461,15 +19156,16 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x1
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB61_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v6, v6, v5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
@@ -18478,12 +19174,15 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off offset:2044 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -18496,6 +19195,7 @@ define void @global_system_atomic_fmin_noret_v2bf16__offset12b_pos__amdgpu_no_fi
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB61_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_set_inst_prefetch_distance 0x2
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index 42a8596598bce..a42ff751eac7d 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9223,58 +9223,114 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16:
; GFX942: ; %bb.0:
@@ -9319,54 +9375,106 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16:
; GFX10: ; %bb.0:
@@ -9616,61 +9724,119 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
}
define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -9717,56 +9883,110 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
@@ -10024,61 +10244,119 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
}
define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX942: ; %bb.0:
@@ -10126,56 +10404,110 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX10: ; %bb.0:
@@ -10433,56 +10765,111 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
}
define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v6, v3
-; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_noret_bf16:
; GFX942: ; %bb.0:
@@ -10526,52 +10913,103 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_not_b32_e32 v6, v3
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16:
; GFX10: ; %bb.0:
@@ -11618,32 +12056,33 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11651,7 +12090,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -11736,40 +12175,41 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
+; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -12337,62 +12777,121 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
}
define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-NEXT: s_mov_b32 s0, 0
-; GFX12-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_not_b32_e32 v4, v4
-; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_storecnt 0x0
-; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX12-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
+; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -12439,56 +12938,110 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_mov_b32 s0, 0
-; GFX11-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-NEXT: v_not_b32_e32 v4, v4
-; GFX11-NEXT: .p2align 6
-; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-TRUE16-NEXT: .p2align 6
+; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
+; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: buffer_gl1_inv
+; GFX11-TRUE16-NEXT: buffer_gl0_inv
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
+; GFX11-FAKE16-NEXT: .p2align 6
+; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: buffer_gl1_inv
+; GFX11-FAKE16-NEXT: buffer_gl0_inv
+; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
index a1130abf89a7a..dd69d47c7baa5 100644
--- a/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-saddr-load.ll
@@ -2409,11 +2409,17 @@ define amdgpu_ps half @global_load_saddr_f16(ptr addrspace(1) inreg %sbase, i32
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: global_load_saddr_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: global_load_u16 v0, v0, s[2:3]
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: global_load_saddr_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3]
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: global_load_u16 v0, v0, s[2:3]
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%load = load half, ptr addrspace(1) %gep0
@@ -2451,11 +2457,17 @@ define amdgpu_ps half @global_load_saddr_f16_immneg128(ptr addrspace(1) inreg %s
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: global_load_saddr_f16_immneg128:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: global_load_u16 v0, v0, s[2:3] offset:-128
-; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: global_load_saddr_f16_immneg128:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[2:3] offset:-128
+; GFX12-GISEL-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX12-GISEL-FAKE16-LABEL: global_load_saddr_f16_immneg128:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: global_load_u16 v0, v0, s[2:3] offset:-128
+; GFX12-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %gep0, i64 -128
diff --git a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
index a8b3702964940..63f473770d720 100644
--- a/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/i1-to-bf16.ll
@@ -33,85 +33,45 @@ define bfloat @v_uitofp_i1_to_bf16(i1 %num) {
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_uitofp_i1_to_bf16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_uitofp_i1_to_bf16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
-; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-TRUE16-LABEL: v_uitofp_i1_to_bf16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_uitofp_i1_to_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-FAKE16-LABEL: v_uitofp_i1_to_bf16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
-; GFX12-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: v_uitofp_i1_to_bf16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%op = uitofp i1 %num to bfloat
ret bfloat %op
}
@@ -231,12 +191,12 @@ define <2 x bfloat> @v_uitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX11-TRUE16-LABEL: v_uitofp_v2i1_to_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
@@ -251,8 +211,11 @@ define <2 x bfloat> @v_uitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_uitofp_v2i1_to_v2bf16:
@@ -288,13 +251,13 @@ define <2 x bfloat> @v_uitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
+; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
@@ -311,8 +274,11 @@ define <2 x bfloat> @v_uitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_uitofp_v2i1_to_v2bf16:
@@ -529,37 +495,40 @@ define <3 x bfloat> @v_uitofp_v3i1_to_v3bf16(<3 x i1> %num) {
; GFX11-TRUE16-LABEL: v_uitofp_v3i1_to_v3bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_uitofp_v3i1_to_v3bf16:
@@ -606,42 +575,45 @@ define <3 x bfloat> @v_uitofp_v3i1_to_v3bf16(<3 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
-; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v6, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_uitofp_v3i1_to_v3bf16:
@@ -926,44 +898,49 @@ define <4 x bfloat> @v_uitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 1, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 1, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_uitofp_v4i1_to_v4bf16:
@@ -1021,51 +998,57 @@ define <4 x bfloat> @v_uitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
-; GFX12-TRUE16-NEXT: v_and_b16 v2.l, 1, v2.l
+; GFX12-TRUE16-NEXT: v_and_b16 v1.h, 1, v2.l
+; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v2.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.h
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v9, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_uitofp_v4i1_to_v4bf16:
@@ -1357,85 +1340,45 @@ define bfloat @v_sitofp_i1_to_bf16(i1 %num) {
; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: v_sitofp_i1_to_bf16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: v_sitofp_i1_to_bf16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
-; GFX11-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-TRUE16-LABEL: v_sitofp_i1_to_bf16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_sitofp_i1_to_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-FAKE16-LABEL: v_sitofp_i1_to_bf16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
-; GFX12-FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: v_sitofp_i1_to_bf16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
+; GFX12-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%op = sitofp i1 %num to bfloat
ret bfloat %op
}
@@ -1555,12 +1498,12 @@ define <2 x bfloat> @v_sitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX11-TRUE16-LABEL: v_sitofp_v2i1_to_v2bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
@@ -1575,8 +1518,11 @@ define <2 x bfloat> @v_sitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_sitofp_v2i1_to_v2bf16:
@@ -1612,13 +1558,13 @@ define <2 x bfloat> @v_sitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
+; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
@@ -1635,8 +1581,11 @@ define <2 x bfloat> @v_sitofp_v2i1_to_v2bf16(<2 x i1> %num) {
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_sitofp_v2i1_to_v2bf16:
@@ -1855,37 +1804,40 @@ define <3 x bfloat> @v_sitofp_v3i1_to_v3bf16(<3 x i1> %num) {
; GFX11-TRUE16-LABEL: v_sitofp_v3i1_to_v3bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_sitofp_v3i1_to_v3bf16:
@@ -1932,42 +1884,45 @@ define <3 x bfloat> @v_sitofp_v3i1_to_v3bf16(<3 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v2.l
-; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX12-TRUE16-NEXT: v_bfe_u32 v3, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v6, v0, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc_lo
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_sitofp_v3i1_to_v3bf16:
@@ -2255,44 +2210,49 @@ define <4 x bfloat> @v_sitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 1, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 1, v2.l
+; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v2.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.h
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_sitofp_v4i1_to_v4bf16:
@@ -2350,51 +2310,57 @@ define <4 x bfloat> @v_sitofp_v4i1_to_v4bf16(<4 x i1> %num) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v3.l
-; GFX12-TRUE16-NEXT: v_and_b16 v2.l, 1, v2.l
+; GFX12-TRUE16-NEXT: v_and_b16 v1.h, 1, v2.l
+; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 1, v1.l
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 1, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 1, v1.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v2.l
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.h
+; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, -1.0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v1.l
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, -1.0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.h
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 1, v0.l
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1.0, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX12-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v8, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v5, v9, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v10, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v4, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.h
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_sitofp_v4i1_to_v4bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index d57229a23b14f..d6338f83271d8 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -1010,14 +1010,14 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX9-SDAG-NEXT: v_pk_mad_u16 v4, v1, v3, v1
; GFX9-SDAG-NEXT: v_pk_mad_u16 v5, v0, v2, v0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v6, v5, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v7, v4, v3
; GFX9-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
-; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1 op_sel_hi:[1,1,0]
; GFX9-SDAG-NEXT: v_pk_mad_u16 v2, v5, v2, 1 op_sel_hi:[1,1,0]
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -1048,14 +1048,14 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX10-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX10-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX10-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
@@ -1086,16 +1086,16 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX11-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX11-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -1136,16 +1136,16 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX1200-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX1200-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX1200-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX1200-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX1200-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX1200-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX1200-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -1187,16 +1187,16 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX1250-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX1250-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX1250-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX1250-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX1250-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX1250-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -2534,14 +2534,14 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX9-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX9-SDAG-NEXT: v_pk_mad_u16 v4, v1, v3, v1
; GFX9-SDAG-NEXT: v_pk_mad_u16 v5, v0, v2, v0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v6, v5, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v7, v4, v3
; GFX9-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
-; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
+; GFX9-SDAG-NEXT: v_pk_mad_u16 v3, v4, v3, 1 op_sel_hi:[1,1,0]
; GFX9-SDAG-NEXT: v_pk_mad_u16 v2, v5, v2, 1 op_sel_hi:[1,1,0]
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -2572,14 +2572,14 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX10-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX10-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX10-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX10-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX10-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v7, v1
@@ -2610,16 +2610,16 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX11-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX11-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX11-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -2660,16 +2660,16 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX1200-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1200-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX1200-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX1200-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX1200-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX1200-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX1200-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX1200-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX1200-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -2711,16 +2711,16 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
; GFX1250-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0]
-; GFX1250-SDAG-NEXT: v_pk_add_u16 v1, v1, 1
+; GFX1250-SDAG-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v4, v0, v2, v0
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v5, v1, v3, v1
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v0, v0, v2, 1 op_sel_hi:[1,1,0]
-; GFX1250-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1
+; GFX1250-SDAG-NEXT: v_pk_mad_u16 v1, v1, v3, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v6, v4, v2
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v7, v5, v3
-; GFX1250-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1
+; GFX1250-SDAG-NEXT: v_pk_mad_u16 v3, v5, v3, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: v_pk_mad_u16 v2, v4, v2, 1 op_sel_hi:[1,1,0]
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-SDAG-NEXT: v_pk_mul_lo_u16 v0, v6, v0
@@ -11804,10 +11804,10 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) {
; GFX11-SDAG-TRUE16-LABEL: multi_use_mul_mad_i16_var:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v2.l, v0.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v2.h, v0.l, v1.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_mad_u16 v0.h, v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: multi_use_mul_mad_i16_var:
@@ -11845,10 +11845,10 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1200-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
+; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v2.l, v0.l, v1.l, v2.l
+; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v2.h, v0.l, v1.l, v3.l
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v2.l
-; GFX1200-SDAG-TRUE16-NEXT: v_mad_u16 v0.h, v0.h, v1.l, v3.l
+; GFX1200-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
; GFX1200-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-FAKE16-LABEL: multi_use_mul_mad_i16_var:
@@ -11912,25 +11912,15 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) {
; GFX1250-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
; GFX1250-GISEL-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX1250-SDAG-REAL16-LABEL: multi_use_mul_mad_i16_var:
-; GFX1250-SDAG-REAL16: ; %bb.0: ; %entry
-; GFX1250-SDAG-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.l, v0.h, v1.l, v2.l
-; GFX1250-SDAG-REAL16-NEXT: v_mad_u16 v0.h, v0.h, v1.l, v3.l
-; GFX1250-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-GISEL-REAL16-LABEL: multi_use_mul_mad_i16_var:
-; GFX1250-GISEL-REAL16: ; %bb.0: ; %entry
-; GFX1250-GISEL-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-REAL16-NEXT: v_mad_u16 v2.l, v0.l, v1.l, v2.l
-; GFX1250-GISEL-REAL16-NEXT: v_mad_u16 v2.h, v0.l, v1.l, v3.l
-; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v0, v2
-; GFX1250-GISEL-REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-REAL16-LABEL: multi_use_mul_mad_i16_var:
+; GFX1250-REAL16: ; %bb.0: ; %entry
+; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: v_mad_u16 v2.l, v0.l, v1.l, v2.l
+; GFX1250-REAL16-NEXT: v_mad_u16 v2.h, v0.l, v1.l, v3.l
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, v2
+; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
; GFX1250-GISEL-LABEL: multi_use_mul_mad_i16_var:
; GFX1250-GISEL: ; %bb.0: ; %entry
; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
@@ -12682,18 +12672,31 @@ define i64 @mul_u24_known_24bit_add64(i16 %x.s, i16 %y.s, i64 %z) {
; GFX11-SDAG-FAKE16-NEXT: v_mad_u64_u32 v[0:1], null, v4, v5, v[2:3]
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-GISEL-LABEL: mul_u24_known_24bit_add64:
-; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_mul_u32_u24_e32 v4, v0, v1
-; GFX11-GISEL-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
-; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX11-GISEL-TRUE16-LABEL: mul_u24_known_24bit_add64:
+; GFX11-GISEL-TRUE16: ; %bb.0:
+; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_u32_u24_e32 v4, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
+; GFX11-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-FAKE16-LABEL: mul_u24_known_24bit_add64:
+; GFX11-GISEL-FAKE16: ; %bb.0:
+; GFX11-GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-FAKE16-NEXT: v_mul_u32_u24_e32 v4, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
+; GFX11-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
+; GFX11-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX11-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1200-SDAG-TRUE16-LABEL: mul_u24_known_24bit_add64:
; GFX1200-SDAG-TRUE16: ; %bb.0:
@@ -12721,23 +12724,41 @@ define i64 @mul_u24_known_24bit_add64(i16 %x.s, i16 %y.s, i64 %z) {
; GFX1200-SDAG-FAKE16-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v1, v[2:3]
; GFX1200-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1200-GISEL-LABEL: mul_u24_known_24bit_add64:
-; GFX1200-GISEL: ; %bb.0:
-; GFX1200-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1200-GISEL-NEXT: s_wait_expcnt 0x0
-; GFX1200-GISEL-NEXT: s_wait_samplecnt 0x0
-; GFX1200-GISEL-NEXT: s_wait_bvhcnt 0x0
-; GFX1200-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX1200-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1200-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-GISEL-NEXT: v_mul_u32_u24_e32 v4, v0, v1
-; GFX1200-GISEL-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
-; GFX1200-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
-; GFX1200-GISEL-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX1200-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1200-GISEL-TRUE16-LABEL: mul_u24_known_24bit_add64:
+; GFX1200-GISEL-TRUE16: ; %bb.0:
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1200-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1200-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX1200-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-GISEL-TRUE16-NEXT: v_mul_u32_u24_e32 v4, v0, v1
+; GFX1200-GISEL-TRUE16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
+; GFX1200-GISEL-TRUE16-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
+; GFX1200-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-GISEL-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1200-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1200-GISEL-FAKE16-LABEL: mul_u24_known_24bit_add64:
+; GFX1200-GISEL-FAKE16: ; %bb.0:
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1200-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1200-GISEL-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX1200-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-GISEL-FAKE16-NEXT: v_mul_u32_u24_e32 v4, v0, v1
+; GFX1200-GISEL-FAKE16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1
+; GFX1200-GISEL-FAKE16-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2
+; GFX1200-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-GISEL-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
+; GFX1200-GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-SDAG-FAKE16-LABEL: mul_u24_known_24bit_add64:
; GFX1250-SDAG-FAKE16: ; %bb.0:
@@ -12775,8 +12796,8 @@ define i64 @mul_u24_known_24bit_add64(i16 %x.s, i16 %y.s, i64 %z) {
; GFX1250-GISEL-REAL16: ; %bb.0:
; GFX1250-GISEL-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-GISEL-REAL16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-REAL16-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_u32_u16_e32 v4, v1.l
; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-GISEL-REAL16-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v4
; GFX1250-GISEL-REAL16-NEXT: v_mul_u32_u24_e32 v0, v0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
index 1ad2f1079b1bb..5796e095d66a5 100644
--- a/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/legalize-amdgcn.raw.ptr.buffer.load.ll
@@ -1495,8 +1495,8 @@ define half @raw_ptr_buffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
- ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
- ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -2240,8 +2240,8 @@ define half @raw_ptr_buffer_load_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr
; GFX1250-NEXT: bb.3:
; GFX1250-NEXT: $exec_lo = S_MOV_B32 [[S_MOV_B32_]]
; GFX1250-NEXT: [[COPY10:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_USHORT_VBUFFER_OFFEN]].lo16
- ; GFX1250-NEXT: $vgpr0 = COPY [[COPY10]]
- ; GFX1250-NEXT: SI_RETURN implicit $vgpr0
+ ; GFX1250-NEXT: $vgpr0_lo16 = COPY [[COPY10]]
+ ; GFX1250-NEXT: SI_RETURN implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.raw.ptr.buffer.load.f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll
index edd9543d861cc..3dd00c7a9d762 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.dead.ll
@@ -645,6 +645,7 @@ define %non_trivial_types @dead_non_trivial(i1 %cond, %non_trivial_types %x, ptr
; ASM-GISEL-TRUE16-NEXT: scratch_load_b32 v68, off, s32 offset:84
; ASM-GISEL-TRUE16-NEXT: v_and_b32_e32 v1, 1, v1
; ASM-GISEL-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; ASM-GISEL-TRUE16-NEXT: ; kill: def $vgpr2_lo16 killed $vgpr2_lo16 def $vgpr2_hi16
; ASM-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; ASM-GISEL-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v1
; ASM-GISEL-TRUE16-NEXT: s_cbranch_execz .LBB3_2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll
index 0122b70985c86..a49dc122ddc37 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.exp2.ll
@@ -162,7 +162,7 @@ define half @v_exp2_f16_uniform(half inreg %src) {
; GFX12-NEXT: v_s_exp_f16 s0, s0
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-NEXT: s_setpc_b64 s[30:31]
%exp2 = call half @llvm.amdgcn.exp2.f16(half %src)
ret half %exp2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
index 87a41bf3e3cc6..f8ed2b40449c8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -100,11 +100,10 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -168,11 +167,10 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -302,11 +300,10 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -572,12 +569,11 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], v[1:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -702,10 +698,9 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
;
; GFX11-TRUE16-LABEL: gather4_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], v[1:2], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -902,5 +897,3 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f32(i32, float, hal
attributes #0 = { nounwind }
attributes #1 = { nounwind readonly }
attributes #2 = { nounwind readnone }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
index 2f288d6fb95a6..8b9b6321d6516 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
@@ -304,10 +304,9 @@ main_body:
define amdgpu_ps <4 x float> @load_2dmsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %fragid) {
; GFX11-TRUE16-LABEL: load_2dmsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -339,11 +338,10 @@ main_body:
define amdgpu_ps <4 x float> @load_2darraymsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX11-TRUE16-LABEL: load_2darraymsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
-; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[3:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
index e3c84dcfcc027..e45eab302d6ac 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
@@ -141,11 +141,10 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX11-TRUE16-NEXT: image_sample v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -209,11 +208,10 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE a16
+; GFX11-TRUE16-NEXT: image_sample v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -343,11 +341,10 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_sample v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -584,11 +581,10 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_sample_cl v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1068,12 +1064,11 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
-; GFX11-TRUE16-NEXT: image_sample_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: image_sample_b_cl v[0:3], v[1:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1297,13 +1292,12 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: sample_d_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], v[4:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], v[2:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1366,12 +1360,11 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: sample_d_3d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[8:9]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[7:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1386,13 +1379,12 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX12PLUS-TRUE16-LABEL: sample_d_3d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12PLUS-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v[7:9]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
+; GFX12PLUS-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v[6:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1572,14 +1564,13 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
;
; GFX11-TRUE16-LABEL: sample_d_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
-; GFX11-TRUE16-NEXT: image_sample_d_cl_g16 v[0:3], v[4:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: image_sample_d_cl_g16 v[0:3], v[3:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1704,12 +1695,11 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_cl_2d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.h, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[7:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[6:7]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
;
@@ -1792,10 +1782,9 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: sample_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: image_sample_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: image_sample_l v[0:3], v[1:2], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2104,12 +2093,11 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[7:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2124,13 +2112,12 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[6:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2175,12 +2162,11 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[7:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
@@ -2195,13 +2181,12 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
+; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[6:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
index 3e9c6f4d24d83..d84879762814e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.d16.dim.ll
@@ -125,47 +125,19 @@ define amdgpu_ps half @image_sample_2d_f16_tfe(<8 x i32> inreg %rsrc, <4 x i32>
; GFX10-NEXT: global_store_dword v4, v1, s[12:13]
; GFX10-NEXT: ; return to shader part epilog
;
-; GFX11-TRUE16-LABEL: image_sample_2d_f16_tfe:
-; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, exec_lo
-; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-TRUE16-NEXT: image_sample v[3:4], v[0:1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D tfe d16
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: global_store_b32 v2, v4, s[12:13]
-; GFX11-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX11-FAKE16-LABEL: image_sample_2d_f16_tfe:
-; GFX11-FAKE16: ; %bb.0: ; %main_body
-; GFX11-FAKE16-NEXT: s_mov_b32 s14, exec_lo
-; GFX11-FAKE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, v4
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
-; GFX11-FAKE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX11-FAKE16-NEXT: image_sample v[0:1], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D tfe d16
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: global_store_b32 v4, v1, s[12:13]
-; GFX11-FAKE16-NEXT: ; return to shader part epilog
-;
-; GFX12PLUS-TRUE16-LABEL: image_sample_2d_f16_tfe:
-; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: s_mov_b32 s14, exec_lo
-; GFX12PLUS-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
-; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12PLUS-TRUE16-NEXT: v_dual_mov_b32 v4, v3 :: v_dual_mov_b32 v3, v2
-; GFX12PLUS-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s14
-; GFX12PLUS-TRUE16-NEXT: image_sample v[3:4], [v0, v1], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D tfe d16
-; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX12PLUS-TRUE16-NEXT: global_store_b32 v2, v4, s[12:13]
-; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
+; GFX11-LABEL: image_sample_2d_f16_tfe:
+; GFX11: ; %bb.0: ; %main_body
+; GFX11-NEXT: s_mov_b32 s14, exec_lo
+; GFX11-NEXT: s_wqm_b32 exec_lo, exec_lo
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v3, v1
+; GFX11-NEXT: v_mov_b32_e32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v5, v4
+; GFX11-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v5
+; GFX11-NEXT: s_and_b32 exec_lo, exec_lo, s14
+; GFX11-NEXT: image_sample v[0:1], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D tfe d16
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: global_store_b32 v4, v1, s[12:13]
+; GFX11-NEXT: ; return to shader part epilog
;
; GFX12-FAKE16-LABEL: image_sample_2d_f16_tfe:
; GFX12-FAKE16: ; %bb.0: ; %main_body
@@ -638,3 +610,7 @@ declare {<4 x half>,i32} @llvm.amdgcn.image.sample.b.2d.v4f16i32.f32.f32(i32, fl
attributes #0 = { nounwind }
attributes #1 = { nounwind readonly }
attributes #2 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11-FAKE16: {{.*}}
+; GFX11-TRUE16: {{.*}}
+; GFX12PLUS-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
index 8bd42ece040d2..cc3da1ed343df 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
@@ -511,11 +511,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l ; encoding: [0x05,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x0f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -530,11 +528,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX13-TRUE16: ; %bb.0: ; %main_body
-; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
+; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l ; encoding: [0x05,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0x2e,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
@@ -582,11 +578,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12-TRUE16: ; %bb.0: ; %main_body
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l ; encoding: [0x05,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x8f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -601,11 +595,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX13-TRUE16: ; %bb.0: ; %main_body
-; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
+; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l ; encoding: [0x05,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0xae,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
index 0f547e619f260..efbe74b6d3c09 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
@@ -359,10 +359,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
@@ -410,10 +409,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
;
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
-; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
+; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
index 8efa1133bc8c1..331dbd0add969 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.interp.inreg.ll
@@ -427,7 +427,7 @@ define amdgpu_ps half @v_interp_f16_imm_params(float inreg %i, float inreg %j) #
; GFX12-TRUE16-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
; GFX12-TRUE16-GISEL-NEXT: s_add_f16 s0, s0, s1
; GFX12-TRUE16-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-TRUE16-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-TRUE16-GISEL-NEXT: ; return to shader part epilog
;
; GFX12-FAKE16-SDAG-LABEL: v_interp_f16_imm_params:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
index 7fc6166024b62..edc84b7dd1665 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.make.buffer.rsrc.ll
@@ -823,12 +823,12 @@ define amdgpu_ps float @general_case_load_with_waterfall(ptr %p, i16 %stride, i6
;
; REAL16-LABEL: name: general_case_load_with_waterfall
; REAL16: bb.0 (%ir-block.0):
- ; REAL16-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; REAL16-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2_lo16, $vgpr3, $vgpr4, $vgpr5
; REAL16-NEXT: {{ $}}
; REAL16-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; REAL16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr4
; REAL16-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; REAL16-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; REAL16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY $vgpr2_lo16
; REAL16-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; REAL16-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr0
; REAL16-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY1]], %subreg.sub1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
index dbc8ded781d17..406d70ab1e95b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.tbuffer.load.d16.ll
@@ -3,8 +3,8 @@
; RUN: llc < %s -mtriple=amdgpu8.10 | FileCheck -enable-var-scope -check-prefixes=PREGFX10-PACKED %s
; RUN: llc < %s -mtriple=amdgpu9.00 | FileCheck -enable-var-scope -check-prefixes=PREGFX10-PACKED %s
; RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck -enable-var-scope -check-prefixes=GFX10-PACKED %s
-; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED,GFX11-PACKED-REAL16 %s
-; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED,GFX11-PACKED-FAKE16 %s
+; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED %s
+; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED %s
define amdgpu_ps half @tbuffer_load_d16_x(ptr addrspace(8) inreg %rsrc) {
; PREGFX10-UNPACKED-LABEL: tbuffer_load_d16_x:
@@ -90,20 +90,6 @@ define amdgpu_ps half @tbuffer_load_d16_xyz(ptr addrspace(8) inreg %rsrc) {
; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, v1
; GFX10-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-REAL16-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED-REAL16: ; %bb.0: ; %main_body
-; GFX11-PACKED-REAL16-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
-; GFX11-PACKED-REAL16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-REAL16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-PACKED-REAL16-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-FAKE16-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED-FAKE16: ; %bb.0: ; %main_body
-; GFX11-PACKED-FAKE16-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
-; GFX11-PACKED-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-PACKED-FAKE16-NEXT: ; return to shader part epilog
main_body:
%data = call <3 x half> @llvm.amdgcn.raw.ptr.tbuffer.load.v3f16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 22, i32 0)
%elt = extractelement <3 x half> %data, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll
index 43994e4956ff0..eb1e23970b287 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.tbuffer.load.d16.ll
@@ -3,10 +3,10 @@
; RUN: llc < %s -mtriple=amdgpu8.10 | FileCheck -enable-var-scope -check-prefixes=PREGFX10-PACKED %s
; RUN: llc < %s -mtriple=amdgpu9.00 | FileCheck -enable-var-scope -check-prefixes=PREGFX10-PACKED %s
; RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck -enable-var-scope -check-prefixes=GFX10-PACKED %s
-; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED,GFX11-PACKED-REAL16 %s
-; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED,GFX11-PACKED-FAKE16 %s
-; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX12-PACKED,GFX12-PACKED-REAL16 %s
-; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX12-PACKED,GFX12-PACKED-FAKE16 %s
+; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED %s
+; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED %s
+; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX12-PACKED %s
+; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck -enable-var-scope -check-prefixes=GFX12-PACKED %s
define amdgpu_ps half @tbuffer_load_d16_x(<4 x i32> inreg %rsrc) {
; PREGFX10-UNPACKED-LABEL: tbuffer_load_d16_x:
@@ -105,34 +105,6 @@ define amdgpu_ps half @tbuffer_load_d16_xyz(<4 x i32> inreg %rsrc) {
; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, v1
; GFX10-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-REAL16-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED-REAL16: ; %bb.0: ; %main_body
-; GFX11-PACKED-REAL16-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
-; GFX11-PACKED-REAL16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-REAL16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-PACKED-REAL16-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-FAKE16-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED-FAKE16: ; %bb.0: ; %main_body
-; GFX11-PACKED-FAKE16-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], 0 format:[BUF_FMT_32_FLOAT]
-; GFX11-PACKED-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-PACKED-FAKE16-NEXT: ; return to shader part epilog
-;
-; GFX12-PACKED-REAL16-LABEL: tbuffer_load_d16_xyz:
-; GFX12-PACKED-REAL16: ; %bb.0: ; %main_body
-; GFX12-PACKED-REAL16-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], null format:[BUF_FMT_32_FLOAT]
-; GFX12-PACKED-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX12-PACKED-REAL16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-PACKED-REAL16-NEXT: ; return to shader part epilog
-;
-; GFX12-PACKED-FAKE16-LABEL: tbuffer_load_d16_xyz:
-; GFX12-PACKED-FAKE16: ; %bb.0: ; %main_body
-; GFX12-PACKED-FAKE16-NEXT: tbuffer_load_d16_format_xyz v[0:1], off, s[0:3], null format:[BUF_FMT_32_FLOAT]
-; GFX12-PACKED-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX12-PACKED-FAKE16-NEXT: ; return to shader part epilog
main_body:
%data = call <3 x half> @llvm.amdgcn.raw.tbuffer.load.v3f16(<4 x i32> %rsrc, i32 0, i32 0, i32 22, i32 0)
%elt = extractelement <3 x half> %data, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
index c876aa657338d..4bb90082100eb 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
@@ -409,24 +409,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, -1
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_and_b32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_and_b32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -447,24 +447,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, -1
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_and_b32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_and_b32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -485,6 +485,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_and_b32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -503,6 +522,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_and_b32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.and.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
index 1f3d19668f86b..d02acf922011b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
@@ -409,24 +409,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, 0
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_or_b32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_or_b32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -447,24 +447,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_or_b32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_or_b32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -485,6 +485,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_or_b32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -503,6 +522,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_or_b32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.or.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
index 69b24c6f1b17a..f0ec88f3f3adf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
@@ -421,24 +421,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, 0
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_max_u32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_max_u32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -459,24 +459,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_max_u32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_max_u32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -497,6 +497,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_max_u32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -515,6 +534,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_max_u32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.umax.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
index 7a3a939891089..c1a92655d7cd2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
@@ -409,24 +409,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, -1
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_min_u32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_min_u32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -447,24 +447,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, -1
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_min_u32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, -1
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_min_u32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -485,6 +485,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_min_u32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -503,6 +522,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_min_u32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.umin.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
index 7d6c296515733..7bb81893b738d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
@@ -503,24 +503,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1164GISEL-LABEL: divergent_value_i16:
-; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1164GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: s_mov_b32 s2, 0
-; GFX1164GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s3, s[0:1]
-; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: v_readlane_b32 s4, v2, s3
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[0:1], s3
-; GFX1164GISEL-NEXT: s_xor_b32 s2, s2, s4
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[0:1], 0
-; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1164GISEL-NEXT: ; %bb.2:
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX1164GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1164GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1164GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1164GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-FAKE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-FAKE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-FAKE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-FAKE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-FAKE16-NEXT: s_xor_b32 s2, s2, s4
+; GFX1164GISEL-FAKE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1164GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1132DAGISEL-FAKE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-FAKE16: ; %bb.0: ; %entry
@@ -541,24 +541,24 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1132GISEL-LABEL: divergent_value_i16:
-; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1132GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1132GISEL-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s0, 0
-; GFX1132GISEL-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s2, s1
-; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s3, v2, s2
-; GFX1132GISEL-NEXT: s_bitset0_b32 s1, s2
-; GFX1132GISEL-NEXT: s_xor_b32 s0, s0, s3
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s1, 0
-; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB1_1
-; GFX1132GISEL-NEXT: ; %bb.2:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v2, s0
-; GFX1132GISEL-NEXT: global_store_b16 v[0:1], v2, off
-; GFX1132GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX1132GISEL-FAKE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-FAKE16: ; %bb.0: ; %entry
+; GFX1132GISEL-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-FAKE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-FAKE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-FAKE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-FAKE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-FAKE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-FAKE16-NEXT: s_xor_b32 s0, s0, s3
+; GFX1132GISEL-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-FAKE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-FAKE16-NEXT: ; %bb.2:
+; GFX1132GISEL-FAKE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1164DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1164DAGISEL-TRUE16: ; %bb.0: ; %entry
@@ -579,6 +579,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1164DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1164DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX1164GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1164GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1164GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1164GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b64 s[0:1], exec
+; GFX1164GISEL-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX1164GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1164GISEL-TRUE16-NEXT: s_ctz_i32_b64 s3, s[0:1]
+; GFX1164GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1164GISEL-TRUE16-NEXT: v_readlane_b32 s4, v2, s3
+; GFX1164GISEL-TRUE16-NEXT: s_bitset0_b64 s[0:1], s3
+; GFX1164GISEL-TRUE16-NEXT: s_xor_b32 s2, s2, s4
+; GFX1164GISEL-TRUE16-NEXT: s_cmp_lg_u64 s[0:1], 0
+; GFX1164GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1164GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1164GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1164GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
; GFX1132DAGISEL-TRUE16-LABEL: divergent_value_i16:
; GFX1132DAGISEL-TRUE16: ; %bb.0: ; %entry
; GFX1132DAGISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -597,6 +616,25 @@ define void @divergent_value_i16(ptr addrspace(1) %out, i16 %in) {
; GFX1132DAGISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
; GFX1132DAGISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
; GFX1132DAGISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1132GISEL-TRUE16-LABEL: divergent_value_i16:
+; GFX1132GISEL-TRUE16: ; %bb.0: ; %entry
+; GFX1132GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1132GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v2, v2.l
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s1, exec_lo
+; GFX1132GISEL-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX1132GISEL-TRUE16-NEXT: .LBB1_1: ; =>This Inner Loop Header: Depth=1
+; GFX1132GISEL-TRUE16-NEXT: s_ctz_i32_b32 s2, s1
+; GFX1132GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1132GISEL-TRUE16-NEXT: v_readlane_b32 s3, v2, s2
+; GFX1132GISEL-TRUE16-NEXT: s_bitset0_b32 s1, s2
+; GFX1132GISEL-TRUE16-NEXT: s_xor_b32 s0, s0, s3
+; GFX1132GISEL-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX1132GISEL-TRUE16-NEXT: s_cbranch_scc1 .LBB1_1
+; GFX1132GISEL-TRUE16-NEXT: ; %bb.2:
+; GFX1132GISEL-TRUE16-NEXT: v_mov_b32_e32 v2, s0
+; GFX1132GISEL-TRUE16-NEXT: global_store_b16 v[0:1], v2, off
+; GFX1132GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
entry:
%result = call i16 @llvm.amdgcn.wave.reduce.xor.i16(i16 %in, i32 1)
store i16 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll
index 332b6ff3693b1..9cc25b453e6a5 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.load.d16.ll
@@ -3,8 +3,8 @@
; RUN: llc < %s -mtriple=amdgpu8.10 | FileCheck -enable-var-scope -check-prefixes=PREGFX10-PACKED %s
; RUN: llc < %s -mtriple=amdgpu9.00 | FileCheck -enable-var-scope -check-prefixes=PREGFX10-PACKED %s
; RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck -enable-var-scope -check-prefixes=GFX10-PACKED %s
-; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED,GFX11-PACKED-REAL16 %s
-; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED,GFX11-PACKED-FAKE16 %s
+; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED %s
+; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED %s
define amdgpu_ps half @tbuffer_load_d16_x(ptr addrspace(8) inreg %rsrc) {
; PREGFX10-UNPACKED-LABEL: tbuffer_load_d16_x:
@@ -101,22 +101,6 @@ define amdgpu_ps half @tbuffer_load_d16_xyz(ptr addrspace(8) inreg %rsrc) {
; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, v1
; GFX10-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-REAL16-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED-REAL16: ; %bb.0: ; %main_body
-; GFX11-PACKED-REAL16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-PACKED-REAL16-NEXT: tbuffer_load_d16_format_xyz v[0:1], v0, s[0:3], 0 format:[BUF_FMT_32_FLOAT] idxen
-; GFX11-PACKED-REAL16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-REAL16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-PACKED-REAL16-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-FAKE16-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED-FAKE16: ; %bb.0: ; %main_body
-; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-PACKED-FAKE16-NEXT: tbuffer_load_d16_format_xyz v[0:1], v0, s[0:3], 0 format:[BUF_FMT_32_FLOAT] idxen
-; GFX11-PACKED-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-PACKED-FAKE16-NEXT: ; return to shader part epilog
main_body:
%data = call <3 x half> @llvm.amdgcn.struct.ptr.tbuffer.load.v3f16(ptr addrspace(8) %rsrc, i32 0, i32 0, i32 0, i32 22, i32 0)
%elt = extractelement <3 x half> %data, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll
index 7c668f3ce9823..ec7e6a09f5ed2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.load.d16.ll
@@ -3,10 +3,10 @@
; RUN: llc < %s -mtriple=amdgpu8.10 | FileCheck -enable-var-scope -check-prefixes=PREGFX10-PACKED %s
; RUN: llc < %s -mtriple=amdgpu9.00 | FileCheck -enable-var-scope -check-prefixes=PREGFX10-PACKED %s
; RUN: llc < %s -mtriple=amdgpu10.10 | FileCheck -enable-var-scope -check-prefixes=GFX10-PACKED %s
-; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED,GFX11-PACKED-REAL16 %s
-; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED,GFX11-PACKED-FAKE16 %s
-; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=+real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX12-PACKED,GFX12-PACKED-REAL16 %s
-; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=-real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX12-PACKED,GFX12-PACKED-FAKE16 %s
+; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=+real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED %s
+; RUN: llc < %s -mtriple=amdgpu11.00 -mattr=-real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX11-PACKED %s
+; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=+real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX12-PACKED %s
+; RUN: llc < %s -mtriple=amdgpu12.00 -mattr=-real-true16 | FileCheck -enable-var-scope -check-prefixes=GFX12-PACKED %s
define amdgpu_ps half @tbuffer_load_d16_x(<4 x i32> inreg %rsrc) {
; PREGFX10-UNPACKED-LABEL: tbuffer_load_d16_x:
@@ -118,38 +118,6 @@ define amdgpu_ps half @tbuffer_load_d16_xyz(<4 x i32> inreg %rsrc) {
; GFX10-PACKED-NEXT: s_waitcnt vmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, v1
; GFX10-PACKED-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-REAL16-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED-REAL16: ; %bb.0: ; %main_body
-; GFX11-PACKED-REAL16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-PACKED-REAL16-NEXT: tbuffer_load_d16_format_xyz v[0:1], v0, s[0:3], 0 format:[BUF_FMT_32_FLOAT] idxen
-; GFX11-PACKED-REAL16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-REAL16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-PACKED-REAL16-NEXT: ; return to shader part epilog
-;
-; GFX11-PACKED-FAKE16-LABEL: tbuffer_load_d16_xyz:
-; GFX11-PACKED-FAKE16: ; %bb.0: ; %main_body
-; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-PACKED-FAKE16-NEXT: tbuffer_load_d16_format_xyz v[0:1], v0, s[0:3], 0 format:[BUF_FMT_32_FLOAT] idxen
-; GFX11-PACKED-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX11-PACKED-FAKE16-NEXT: ; return to shader part epilog
-;
-; GFX12-PACKED-REAL16-LABEL: tbuffer_load_d16_xyz:
-; GFX12-PACKED-REAL16: ; %bb.0: ; %main_body
-; GFX12-PACKED-REAL16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-PACKED-REAL16-NEXT: tbuffer_load_d16_format_xyz v[0:1], v0, s[0:3], null format:[BUF_FMT_32_FLOAT] idxen
-; GFX12-PACKED-REAL16-NEXT: s_wait_loadcnt 0x0
-; GFX12-PACKED-REAL16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-PACKED-REAL16-NEXT: ; return to shader part epilog
-;
-; GFX12-PACKED-FAKE16-LABEL: tbuffer_load_d16_xyz:
-; GFX12-PACKED-FAKE16: ; %bb.0: ; %main_body
-; GFX12-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-PACKED-FAKE16-NEXT: tbuffer_load_d16_format_xyz v[0:1], v0, s[0:3], null format:[BUF_FMT_32_FLOAT] idxen
-; GFX12-PACKED-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-PACKED-FAKE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX12-PACKED-FAKE16-NEXT: ; return to shader part epilog
main_body:
%data = call <3 x half> @llvm.amdgcn.struct.tbuffer.load.v3f16(<4 x i32> %rsrc, i32 0, i32 0, i32 0, i32 22, i32 0)
%elt = extractelement <3 x half> %data, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
index e17e1c621da2f..5c1fa3a205d04 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
@@ -85,11 +85,10 @@ define amdgpu_ps <3 x float> @sample_gather(<8 x i32> inreg %rsrc, <4 x i32> inr
define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, <8 x i32> inreg %rsrc2, i16 %s.16, i16 %t.16, i16 %fragid) {
; GFX11-TRUE16-LABEL: sample_load:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -105,11 +104,10 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX1150-TRUE16-LABEL: sample_load:
; GFX1150-TRUE16: ; %bb.0:
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1150-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -159,11 +157,10 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg %samp, <8 x i32> inreg %rsrc2, i16 %s.16, i16 %t.16, i16 %fragid) {
; GFX11-TRUE16-LABEL: load_sample:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -179,11 +176,10 @@ define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX1150-TRUE16-LABEL: load_sample:
; GFX1150-TRUE16: ; %bb.0:
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v1.h, v1.l
+; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
+; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[1:2], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX1150-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
index cbf54cff0fec4..60dba22325d53 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
@@ -1036,10 +1036,11 @@ define <2 x i1> @isnan_v2bf16(<2 x bfloat> %x) nounwind {
; GFX11SELDAG-TRUE16-LABEL: isnan_v2bf16:
; GFX11SELDAG-TRUE16: ; %bb.0:
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v0
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.l
+; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.h
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1119,14 +1120,17 @@ define <3 x i1> @isnan_v3bf16(<3 x bfloat> %x) nounwind {
; GFX11SELDAG-TRUE16-LABEL: isnan_v3bf16:
; GFX11SELDAG-TRUE16: ; %bb.0:
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v2, 0x7fff7fff, v0
-; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0x7fff7fff, v1
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2.l
+; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2.h
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3.l
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-FAKE16-LABEL: isnan_v3bf16:
@@ -1218,16 +1222,21 @@ define <4 x i1> @isnan_v4bf16(<4 x bfloat> %x) nounwind {
; GFX11SELDAG-TRUE16-LABEL: isnan_v4bf16:
; GFX11SELDAG-TRUE16: ; %bb.0:
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v3, 0x7fff7fff, v0
-; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v4, 0x7fff7fff, v1
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3.l
+; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX11SELDAG-TRUE16-NEXT: v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v0.l
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v4.l
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3.h
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v1.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v4.h
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v3.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 0x7f80, v2.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-FAKE16-LABEL: isnan_v4bf16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
index 9d1169f5b2ade..d6c5c76bae327 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
@@ -1409,10 +1409,10 @@ define <2 x i1> @isnan_v2f16(<2 x half> %x) nounwind {
; GFX11SELDAG-TRUE16-LABEL: isnan_v2f16:
; GFX11SELDAG-TRUE16: ; %bb.0:
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.l
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.h, v0.h
-; GFX11SELDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1431,11 +1431,9 @@ define <2 x i1> @isnan_v2f16(<2 x half> %x) nounwind {
; GFX11GLISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11GLISEL-TRUE16-NEXT: v_dual_mov_b32 v1, 3 :: v_dual_mov_b32 v2, 3
; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.l, v1.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.h, v3.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s0, v0.h, v2.l
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0
; GFX11GLISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11GLISEL-FAKE16-LABEL: isnan_v2f16:
@@ -1564,13 +1562,15 @@ define <3 x i1> @isnan_v3f16(<3 x half> %x) nounwind {
; GFX11SELDAG-TRUE16-LABEL: isnan_v3f16:
; GFX11SELDAG-TRUE16: ; %bb.0:
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.l
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.h, v0.h
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l
-; GFX11SELDAG-TRUE16-NEXT: v_dual_mov_b32 v0, v4 :: v_dual_mov_b32 v1, v3
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v2.l, v2.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-FAKE16-LABEL: isnan_v3f16:
@@ -1592,14 +1592,11 @@ define <3 x i1> @isnan_v3f16(<3 x half> %x) nounwind {
; GFX11GLISEL-TRUE16-NEXT: v_dual_mov_b32 v2, 3 :: v_dual_mov_b32 v3, 3
; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v4, 3
; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.l, v2.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.h, v3.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v4
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v1.l, v5.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v1, v3
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s0, v0.h, v3.l
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s1, v1.l, v4.l
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, s1
; GFX11GLISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11GLISEL-FAKE16-LABEL: isnan_v3f16:
@@ -1753,14 +1750,18 @@ define <4 x i1> @isnan_v4f16(<4 x half> %x) nounwind {
; GFX11SELDAG-TRUE16: ; %bb.0:
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.l
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11SELDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc_lo
; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.l, v1.l
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.h, v0.h
-; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v1.h, v1.h
-; GFX11SELDAG-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v4
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v3.l, v3.l
; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v2.l, v2.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX11SELDAG-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX11SELDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
; GFX11SELDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11SELDAG-FAKE16-LABEL: isnan_v4f16:
@@ -1784,17 +1785,13 @@ define <4 x i1> @isnan_v4f16(<4 x half> %x) nounwind {
; GFX11GLISEL-TRUE16-NEXT: v_dual_mov_b32 v2, 3 :: v_dual_mov_b32 v3, 3
; GFX11GLISEL-TRUE16-NEXT: v_dual_mov_b32 v4, 3 :: v_dual_mov_b32 v5, 3
; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.l, v2.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v0.h, v3.l
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v1.l, v6.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v4
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
-; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, v1.h, v7.l
-; GFX11GLISEL-TRUE16-NEXT: v_mov_b32_e32 v1, v5
-; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s0, v0.h, v3.l
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s1, v1.l, v4.l
+; GFX11GLISEL-TRUE16-NEXT: v_cmp_class_f16_e64 s2, v1.h, v5.l
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v0.l, 0, 1, vcc_lo
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 1, s0
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 1, s1
+; GFX11GLISEL-TRUE16-NEXT: v_cndmask_b16 v3.l, 0, 1, s2
; GFX11GLISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11GLISEL-FAKE16-LABEL: isnan_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
index 17f7a401e96c8..6398e85e5ff44 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
@@ -366,15 +366,25 @@ define <2 x i1> @isnan_v2f32(<2 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v2f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v2f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v2f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <2 x i1> @llvm.is.fpclass.v2f32(<2 x float> %x, i32 3) ; nan
ret <2 x i1> %1
}
@@ -424,18 +434,31 @@ define <3 x i1> @isnan_v3f32(<3 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v3f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v3f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v3f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <3 x i1> @llvm.is.fpclass.v3f32(<3 x float> %x, i32 3) ; nan
ret <3 x i1> %1
}
@@ -493,20 +516,35 @@ define <4 x i1> @isnan_v4f32(<4 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v4f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v4f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v4f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <4 x i1> @llvm.is.fpclass.v4f32(<4 x float> %x, i32 3) ; nan
ret <4 x i1> %1
}
@@ -572,23 +610,38 @@ define <5 x i1> @isnan_v5f32(<5 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v5f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v5f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v5f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <5 x i1> @llvm.is.fpclass.v5f32(<5 x float> %x, i32 3) ; nan
ret <5 x i1> %1
}
@@ -662,25 +715,42 @@ define <6 x i1> @isnan_v6f32(<6 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v6f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v5, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v6f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v6f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s5, v5, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: v_cndmask_b16 v5.l, 0, 1, s5
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <6 x i1> @llvm.is.fpclass.v6f32(<6 x float> %x, i32 3) ; nan
ret <6 x i1> %1
}
@@ -762,28 +832,47 @@ define <7 x i1> @isnan_v7f32(<7 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v7f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v5, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v6, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v7f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v6, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v7f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s5, v5, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s6, v6, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: v_cndmask_b16 v5.l, 0, 1, s5
+; GFX11GLISEL-NEXT: v_cndmask_b16 v6.l, 0, 1, s6
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <7 x i1> @llvm.is.fpclass.v7f32(<7 x float> %x, i32 3) ; nan
ret <7 x i1> %1
}
@@ -873,30 +962,51 @@ define <8 x i1> @isnan_v8f32(<8 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v8f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v5, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v6, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v7, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v8f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v6, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v7, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v8f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s5, v5, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s6, v6, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s7, v7, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: v_cndmask_b16 v5.l, 0, 1, s5
+; GFX11GLISEL-NEXT: v_cndmask_b16 v6.l, 0, 1, s6
+; GFX11GLISEL-NEXT: v_cndmask_b16 v7.l, 0, 1, s7
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <8 x i1> @llvm.is.fpclass.v8f32(<8 x float> %x, i32 3) ; nan
ret <8 x i1> %1
}
@@ -1050,50 +1160,87 @@ define <16 x i1> @isnan_v16f32(<16 x float> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v15, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnan_v16f32:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v0, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v1, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v2, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v3, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v4, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v5, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v6, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v7, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v8, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v9, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v10, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v11, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v12, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v13, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v14, 3
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f32_e64 s0, v15, 3
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnan_v16f32:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v1, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v2, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v3, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v4, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v5, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v6, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v7, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v8, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v9, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v10, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v11, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v12, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v13, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v14, 3
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s0, v15, 3
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnan_v16f32:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s0, v0, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s1, v1, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s2, v2, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s3, v3, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s4, v4, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s5, v5, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s6, v6, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s7, v7, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s8, v8, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s9, v9, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s10, v10, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s11, v11, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s12, v12, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s13, v13, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s14, v14, 3
+; GFX11GLISEL-NEXT: v_cmp_class_f32_e64 s15, v15, 3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: v_cndmask_b16 v2.l, 0, 1, s2
+; GFX11GLISEL-NEXT: v_cndmask_b16 v3.l, 0, 1, s3
+; GFX11GLISEL-NEXT: v_cndmask_b16 v4.l, 0, 1, s4
+; GFX11GLISEL-NEXT: v_cndmask_b16 v5.l, 0, 1, s5
+; GFX11GLISEL-NEXT: v_cndmask_b16 v6.l, 0, 1, s6
+; GFX11GLISEL-NEXT: v_cndmask_b16 v7.l, 0, 1, s7
+; GFX11GLISEL-NEXT: v_cndmask_b16 v8.l, 0, 1, s8
+; GFX11GLISEL-NEXT: v_cndmask_b16 v9.l, 0, 1, s9
+; GFX11GLISEL-NEXT: v_cndmask_b16 v10.l, 0, 1, s10
+; GFX11GLISEL-NEXT: v_cndmask_b16 v11.l, 0, 1, s11
+; GFX11GLISEL-NEXT: v_cndmask_b16 v12.l, 0, 1, s12
+; GFX11GLISEL-NEXT: v_cndmask_b16 v13.l, 0, 1, s13
+; GFX11GLISEL-NEXT: v_cndmask_b16 v14.l, 0, 1, s14
+; GFX11GLISEL-NEXT: v_cndmask_b16 v15.l, 0, 1, s15
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <16 x i1> @llvm.is.fpclass.v16f32(<16 x float> %x, i32 3) ; nan
ret <16 x i1> %1
}
@@ -1473,15 +1620,25 @@ define <2 x i1> @isnormal_v2f64(<2 x double> %x) nounwind {
; GFX10CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s4
; GFX10CHECK-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11CHECK-LABEL: isnormal_v2f64:
-; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11CHECK-NEXT: v_cmp_class_f64_e64 s0, v[0:1], 0x108
-; GFX11CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11CHECK-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 0x108
-; GFX11CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11CHECK-NEXT: s_setpc_b64 s[30:31]
+; GFX11SELDAG-LABEL: isnormal_v2f64:
+; GFX11SELDAG: ; %bb.0:
+; GFX11SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11SELDAG-NEXT: v_cmp_class_f64_e64 s0, v[0:1], 0x108
+; GFX11SELDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11SELDAG-NEXT: v_cmp_class_f64_e64 s0, v[2:3], 0x108
+; GFX11SELDAG-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11SELDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11GLISEL-LABEL: isnormal_v2f64:
+; GFX11GLISEL: ; %bb.0:
+; GFX11GLISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11GLISEL-NEXT: v_cmp_class_f64_e64 s0, v[0:1], 0x108
+; GFX11GLISEL-NEXT: v_cmp_class_f64_e64 s1, v[2:3], 0x108
+; GFX11GLISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11GLISEL-NEXT: v_cndmask_b16 v0.l, 0, 1, s0
+; GFX11GLISEL-NEXT: v_cndmask_b16 v1.l, 0, 1, s1
+; GFX11GLISEL-NEXT: s_setpc_b64 s[30:31]
%1 = call <2 x i1> @llvm.is.fpclass.v2f64(<2 x double> %x, i32 264) ; 0x108 = "normal"
ret <2 x i1> %1
}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
index ce6e8f09e6676..ce073335f8912 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.maximum.f16.ll
@@ -548,9 +548,9 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
; GFX1170-TRUE16-LABEL: v_maximum_f16__nnan_src1:
; GFX1170-TRUE16: ; %bb.0:
; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-TRUE16-NEXT: v_add_f16_e32 v1.l, 1.0, v1.l
+; GFX1170-TRUE16-NEXT: v_add_f16_e32 v0.h, 1.0, v1.l
; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v1.l
+; GFX1170-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v0.h
; GFX1170-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-FAKE16-LABEL: v_maximum_f16__nnan_src1:
@@ -568,9 +568,9 @@ define half @v_maximum_f16__nnan_src1(half %src0, half %arg1) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, 1.0, v1.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v0.h, 1.0, v1.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v0.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximum_f16__nnan_src1:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
index cccfb797a1549..0eec3135d1901 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.minimum.f16.ll
@@ -476,9 +476,9 @@ define half @v_minimum_f16__nnan_src1(half %src0, half %arg1) {
; GFX1170-TRUE16-LABEL: v_minimum_f16__nnan_src1:
; GFX1170-TRUE16: ; %bb.0:
; GFX1170-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1170-TRUE16-NEXT: v_add_f16_e32 v1.l, 1.0, v1.l
+; GFX1170-TRUE16-NEXT: v_add_f16_e32 v0.h, 1.0, v1.l
; GFX1170-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1170-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v1.l
+; GFX1170-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v0.h
; GFX1170-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1170-FAKE16-LABEL: v_minimum_f16__nnan_src1:
@@ -496,9 +496,9 @@ define half @v_minimum_f16__nnan_src1(half %src0, half %arg1) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_add_f16_e32 v1.l, 1.0, v1.l
+; GFX12-TRUE16-NEXT: v_add_f16_e32 v0.h, 1.0, v1.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v0.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimum_f16__nnan_src1:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index b04bd5d4de305..d78115c75cd0b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -443,7 +443,7 @@ define amdgpu_ps half @sqrt_f16_s(half inreg %x) {
; GFX12-TRUE16-SDAG-NEXT: v_s_sqrt_f16 s0, s0
; GFX12-TRUE16-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-TRUE16-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX12-TRUE16-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-TRUE16-GISEL-LABEL: sqrt_f16_s:
@@ -451,7 +451,7 @@ define amdgpu_ps half @sqrt_f16_s(half inreg %x) {
; GFX12-TRUE16-GISEL-NEXT: v_s_sqrt_f16 s0, s0
; GFX12-TRUE16-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-TRUE16-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-TRUE16-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-TRUE16-GISEL-NEXT: ; return to shader part epilog
;
; GFX12-FAKE16-LABEL: sqrt_f16_s:
diff --git a/llvm/test/CodeGen/AMDGPU/lround.ll b/llvm/test/CodeGen/AMDGPU/lround.ll
index 887735af85b7d..330f63e64d297 100644
--- a/llvm/test/CodeGen/AMDGPU/lround.ll
+++ b/llvm/test/CodeGen/AMDGPU/lround.ll
@@ -838,16 +838,15 @@ define half @intrinsic_fround_half(half %arg) {
; GFX11-SDAG-TRUE16-LABEL: intrinsic_fround_half:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_trunc_f16_e32 v1.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_sub_f16_e32 v1.l, v1.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_trunc_f16_e32 v0.h, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_sub_f16_e32 v1.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ge_f16_e64 s0, |v1.l|, 0.5
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s0
+; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v1, 0x7fff, v1, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: intrinsic_fround_half:
@@ -963,18 +962,17 @@ define i32 @intrinsic_lround_i32_f16(half %arg) {
; GFX11-SDAG-TRUE16-LABEL: intrinsic_lround_i32_f16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_trunc_f16_e32 v1.h, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_sub_f16_e32 v1.l, v1.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_trunc_f16_e32 v0.h, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_sub_f16_e32 v1.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_cmp_ge_f16_e64 s0, |v1.l|, 0.5
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v0, 0x7fff, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, 0, 0x3c00, s0
+; GFX11-SDAG-TRUE16-NEXT: v_bfi_b32 v1, 0x7fff, v1, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.h, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
index 10c55a59f84d4..1e7e9b9b05f86 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
@@ -46,22 +46,14 @@ define <2 x bfloat> @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_constlo(bfloat %src0,
}
define <2 x bfloat> @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_reglo(bfloat %src0, bfloat %src1, bfloat %src2, bfloat %lo) #0 {
-; GFX1250-FAKE16-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_reglo:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_fma_mixhi_bf16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, v3
-; GFX1250-FAKE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250-REAL16-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_reglo:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_reglo:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_fma_mixhi_bf16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_mov_b32_e32 v0, v3
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%src0.ext = fpext bfloat %src0 to float
%src1.ext = fpext bfloat %src1 to float
%src2.ext = fpext bfloat %src2 to float
@@ -176,11 +168,10 @@ define <2 x bfloat> @v_mad_mixhi_bf16_bf16lo_bf16lo_bf16lo_undeflo_clamp_postcvt
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v1, v0, v1, v2 op_sel_hi:[1,1,1]
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_fma_mixlo_bf16 v0, v3, v1, v2 op_sel_hi:[1,1,1]
-; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v3, v1, v2 op_sel_hi:[1,1,1] clamp
-; GFX1250-REAL16-NEXT: global_store_b16 v[0:1], v0, off scope:SCOPE_SYS
+; GFX1250-REAL16-NEXT: v_fma_mixhi_bf16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; GFX1250-REAL16-NEXT: global_store_d16_hi_b16 v[0:1], v1, off scope:SCOPE_SYS
; GFX1250-REAL16-NEXT: s_wait_storecnt 0x0
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
%src0.ext = fpext bfloat %src0 to float
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
index db028d4364f35..146a21ec8b0ce 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -180,20 +180,13 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo(half %src0, half %s
}
define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src1, half %src2, half %lo) #0 {
-; SDAG-GFX11-TRUE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
-; SDAG-GFX11-TRUE16: ; %bb.0:
-; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; SDAG-GFX11-FAKE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
-; SDAG-GFX11-FAKE16: ; %bb.0:
-; SDAG-GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-FAKE16-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v3
-; SDAG-GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_mov_b32_e32 v0, v3
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
; GFX9: ; %bb.0:
@@ -226,14 +219,6 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src
; SDAG-CI-NEXT: v_or_b32_e32 v0, v1, v0
; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; GISEL-GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
-; GISEL-GFX11: ; %bb.0:
-; GISEL-GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX11-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, v3
-; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
-;
; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -634,11 +619,10 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi
; SDAG-GFX11-TRUE16-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo_clamp_postcvt_multi_use:
; SDAG-GFX11-TRUE16: ; %bb.0:
; SDAG-GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v1, v0, v1, v2 op_sel_hi:[1,1,1]
; SDAG-GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; SDAG-GFX11-TRUE16-NEXT: v_fma_mixlo_f16 v0, v3, v1, v2 op_sel_hi:[1,1,1]
-; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v3, v1, v2 op_sel_hi:[1,1,1] clamp
-; SDAG-GFX11-TRUE16-NEXT: global_store_b16 v[0:1], v0, off dlc
+; SDAG-GFX11-TRUE16-NEXT: v_fma_mixhi_f16 v0, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v[0:1], v1, off dlc
; SDAG-GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
; SDAG-GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index 5ae87eabda47f..c90d9909ddc4f 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -107,26 +107,27 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -161,17 +162,19 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
@@ -290,8 +293,11 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
@@ -326,8 +332,11 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
@@ -543,19 +552,20 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v3.l, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
@@ -633,21 +643,22 @@ define <2 x bfloat> @v_maximumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
@@ -856,24 +867,26 @@ define <2 x bfloat> @v_maximumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v2bf16_nnan:
@@ -916,26 +929,28 @@ define <2 x bfloat> @v_maximumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v2bf16_nnan:
@@ -1223,58 +1238,58 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) #1 {
; GFX11-TRUE16-LABEL: v_maximumnum_v3bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v9
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v9, v8
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v6.l, s2
; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16:
@@ -1342,62 +1357,65 @@ define <3 x bfloat> @v_maximumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v9
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v11
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v9, v8
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v6.l, s2
; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16:
@@ -1647,30 +1665,32 @@ define <3 x bfloat> @v_maximumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s4, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16_nnan:
@@ -1724,33 +1744,33 @@ define <3 x bfloat> @v_maximumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s4, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16_nnan:
@@ -2128,65 +2148,69 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) #1 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.l, s5
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16:
@@ -2278,74 +2302,81 @@ define <4 x bfloat> @v_maximumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v4.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.l, s5
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16:
@@ -2667,40 +2698,44 @@ define <4 x bfloat> @v_maximumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s0, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.h, s5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v8
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
+; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v6.l, s5
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16_nnan:
@@ -2765,45 +2800,49 @@ define <4 x bfloat> @v_maximumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v6
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v9, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v8
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s0, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v0.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
+; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.h, s5
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v6.l, s5
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16_nnan:
@@ -3350,95 +3389,101 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) #1 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v5.h, v7.l, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v11, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.h, v4.h, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v8, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v12, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v7.l, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v6.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v9, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v7.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v9.l, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.l, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v14.l, v13.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v8.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v6, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v14, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v10.l, v9.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v8.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v18, v19
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v13.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v11
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v14, v13
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v16, v15
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v11
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s0, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v6.l, s7
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v9.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v9.l, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v12.l, s6
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v6bf16:
@@ -3567,107 +3612,115 @@ define <6 x bfloat> @v_maximumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v5.h, v7.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v11, v11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.h, v4.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.l, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v8, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v12, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v14.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v7.l, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v6.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v14, v14
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s5, v9, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v8.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v6, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v14, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v10.l, v9.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v7.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v9.l, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v8.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v12, v11
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v14, v13
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v16, v15
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v18, v19
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v13.l, v12.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v14, v13
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v16, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v11
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v9.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s0, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v6.l, s7
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v9.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v9.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v12.l, s6
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v6bf16:
@@ -4447,126 +4500,139 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) #1 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.h, v7.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v10.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v19, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v15.l, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v13.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v17
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v12.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v9.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.h, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v15, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v13.l, s0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v14.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v13.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v15.l, v12.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v9.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v6.h, v8.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v12.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v3.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v10.l, v13.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v15, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v14.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v8.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v14, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v16.l, v8.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v9.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v1.h, v5.h, s4
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v15, v14
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v8.l
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v12.l, v8.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v0.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v13.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v11.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v13
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v11.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s0, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s8, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v11.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v12.l, v10.l, s8
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v11.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v8.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v17.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, s5
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v8bf16:
@@ -4725,146 +4791,161 @@ define <8 x bfloat> @v_maximumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v9.l, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v9.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v6.h, v8.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v19, v19
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v14, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v15.l, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v13.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v12.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v16.l, v8.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v16
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v15, v17
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v18
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v13.l, s0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v14.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v13.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v15.l, v12.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v14.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v12.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s0, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v3.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v9.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v1.h, v5.h, s4
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v10.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v8.l
-; GFX12-TRUE16-NEXT: s_and_b32 s7, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v15, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v12.l, v8.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v14.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v0.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v13.l, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v11, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v11.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v13
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v11, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v13, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v15, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v11.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s0, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s8, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v11.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v0.l
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, s6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v12.l, v10.l, s8
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v11.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v8.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v17.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s4, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v8
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v8bf16:
@@ -6280,253 +6361,276 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) #
; GFX11-TRUE16-LABEL: v_maximumnum_v16bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v7.h, v15.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v22
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v17.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v23.l, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v19.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s1, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v19.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v17.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v26.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v24, v25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v20.l, v21.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v22.l, v23.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v18.l, v19.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v21.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v19, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v24.l, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v18.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v23.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v19.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v17
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s0, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v26, v27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v19
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v18.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.h, v14.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v17.l, v18.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.h, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v21.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v16.l, v18.l, s2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v14
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s3, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v20, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v5.h, v13.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v18.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v26, v27
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v28.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v22.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v7.l, v21.l, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v26, v27
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s0, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v22.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v24.l, v28.l, s3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v17.l, v23.l, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.l, v18.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v28.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v24.l, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v19.l, v22.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v21.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v28.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v18.l, v17.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v23
; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v21.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v19.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v7
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v22
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v20, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v17.l, v21.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v22
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v15.l, v17.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v16.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v12.h, v17.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v19.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, s2
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v3.h, v11.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v19.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v15.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v20, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v21, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v17.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v21
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v15.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v13.l, v5.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v19.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v1.h, v9.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v9.h, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v19.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v0.h, v8.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v8.h, v15.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v10.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v9.l, v1.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v19, v18
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v14.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v18
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v15.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v19
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v17.l, v16.l, s9
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v13.l, v15.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.l, v3.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v12.l, v4.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v12.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v21, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v1.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v11.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v9.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v10.l, v4.l, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v19 :: v_dual_mov_b32 v5, v18
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v17
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v16bf16:
@@ -6852,294 +6956,320 @@ define <16 x bfloat> @v_maximumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) #
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v15
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v15
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v7.h, v15.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v17
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v17.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v17.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v18.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v18.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v23.l, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v25
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v19.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s1, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v19.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v17.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v26.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v24, v25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v20.l, v21.l, s2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v22.l, v23.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v18.l, v19.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v21.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v19, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v7.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v24.l, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v10
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.h, v14.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v18.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v17.l, v18.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v23.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v19.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v17
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s0, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.h, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v20, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v17
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v21.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v19
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v16.l, v18.l, s2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v14
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v13
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s3, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v20, v19
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v5.h, v13.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v26, v27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v19
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v18.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v21.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v19.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v18.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v26, v27
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v28.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v22.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v23.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v20, v18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v7.l, v21.l, s3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v26, v27
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s0, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v22.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v24.l, v28.l, s3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v17.l, v21.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v22
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v17.l, v23.l, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.l, v18.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v28.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v5.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v24.l, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v12.h, v17.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v19.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s1, s2
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v19.l, v22.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v21.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v3.h, v11.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v19.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v28.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v20, v19
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v21, v18
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v17.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v22, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v18.l, v17.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v18, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v15.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v17.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v17, v14
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v15.l, v17.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v16.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v19.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v1.h, v9.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v21
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v5.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v9.h, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v19.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v15, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s1, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v21, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v0.h, v8.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v10.l, v2.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v8.h, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v9.l, v1.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v19, v18
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v21, v20
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v12.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v12, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v14, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v21, v16
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v14.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v16.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v18
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v15.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v19
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v17.l, v16.l, s9
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v13.l, v15.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s5, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.l, v3.l, s8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v12.l, v4.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v11.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v9.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v10.l, v4.l, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v19 :: v_dual_mov_b32 v5, v18
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v17
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v16bf16:
@@ -9965,494 +10095,534 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v34, v34
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v35, v35
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v48, v48
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v36, v36
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v30
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v29
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v13
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v28
; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v49, v49
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v50, v50
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v51, v51
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v54, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v68, v68
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v30.h, v32.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v37, v37
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v52, v52
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v55, v55
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v66, v66
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v69, v69
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s59, v114, v114
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v51
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v130, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v51, v51
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v53, v53
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v26
; GFX11-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v115, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v64, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 16, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v101, 0xffff0000, v22
; GFX11-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v38, v38
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v65, v65
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v67, v67
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v80, v80
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v27.h, v35.l, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v29
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v52, v114
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v53, v53
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v70, v70
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s60, v115, v115
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v30
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v103, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v33, v33
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v39, v39
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v102, v102
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v71, v71
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v83, v83
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v84, v84
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v133, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v146, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v52, v52
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v54, v54
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v29
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v66, v66
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v98, v98
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v130, v130
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v35.l, v33.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v37.l, v34.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v39.l, v36.l, s4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 16, v25
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 16, v9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 16, v23
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v97, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 16, v21
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 16, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v113, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v55, v55
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v69, v69
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v27
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v70, v70
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v86, v86
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v101, v101
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v102, v102
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v133, v133
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v146, v146
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v54, v54
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v66, v66
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v100.l, v99.l, s16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v132.l, v131.l, s24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v35.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v37.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v39.l, s5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v118, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v81, v81
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v97, v97
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v113, v113
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v114, v114
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v51, v51
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v53, v53
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v55, v55
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v69, v69
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v80.l, v71.l, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v96.l, v87.l, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v112.l, v103.l, s18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v48.l, v147.l, s28
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v26.h, v36.l, s18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v65
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v68
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v64, v117
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v32.l, s20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s16
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v53, v115
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v103, v103
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s47
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v29.h, v33.l, s6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v85, v85
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v24.h, v38.l, s26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v71
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v66, v118
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v69, v128
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v51
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v102.l, v21.h, v49.l, s56
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v54
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v86, v86
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v32.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v26
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v80, v130
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v65.l, v34.l, s24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v29
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v69
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v70, v129
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v30
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v25.h, v37.l, s22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v49
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v102
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v55, v116
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v87, v87
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s41
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s45
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v13.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v24
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v81, v131
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v53
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v80
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s20, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s57, v112, v112
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v70
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v51.l, v32.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s56, v103, v52
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.l, v33.l, s22
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s43
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v14.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v34.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s42
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s46
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v85
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v87, v135
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s29
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v81
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s23, s4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s58, v113, v113
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s57
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v96.l, v23.h, v39.l, s40
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v71.l, v36.l, s28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v52
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v86, v134
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s43
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v26
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s21, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s24, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s58
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v53.l, v34.l, s5
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v33.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v39
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v96
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v84, v133
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s27
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v71
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v10.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v99.l, v22.h, v48.l, s44
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s45, v98, v145
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s47, v101, v147
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v24
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v86
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v97, v144
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s41
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v27
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s22, s3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v48
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v99
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s45
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s47
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v98
-; GFX11-TRUE16-NEXT: s_and_b32 s10, s29, s10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v52.l, v33.l, s3
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s57, v112, v114
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s10
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v11.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v96.l, v39.l, s44
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v84
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s46, v100, v146
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v68.l, v35.l, s26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v102.l, v49.l, s56
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v22
-; GFX11-TRUE16-NEXT: s_and_b32 s14, s43, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s59
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s57
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s12
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v9.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v96
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s27, s8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v7.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v100
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s47, 0, v102
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v113.l, v20.h, v50.l, s60
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v21
-; GFX11-TRUE16-NEXT: s_and_b32 s12, s41, s12
-; GFX11-TRUE16-NEXT: s_and_b32 s16, s45, s16
-; GFX11-TRUE16-NEXT: s_and_b32 s18, s47, s18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s12
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s57, 0, v112
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v113
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v50
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v37
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v82
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v68
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v12.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v67, v119
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v49.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v83, v132
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s56, 0, v103
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v35.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v82.l, v37.l, s40
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v85.l, v38.l, s42
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v83
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v55
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v99.l, v48.l, s46
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v36.l
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s26, s7
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v82
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v54.l, v35.l, s7
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v85
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v64
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v65
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s25, s6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v66
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v67
-; GFX11-TRUE16-NEXT: s_and_b32 s9, s28, s9
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v37.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v38.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v39.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v48.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v87
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v97
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v99
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s46, 0, v101
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v55.l, v36.l, s9
-; GFX11-TRUE16-NEXT: s_and_b32 s11, s40, s11
-; GFX11-TRUE16-NEXT: s_and_b32 s13, s42, s13
-; GFX11-TRUE16-NEXT: s_and_b32 s15, s44, s15
-; GFX11-TRUE16-NEXT: s_and_b32 s17, s46, s17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v64.l, v37.l, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v65.l, v38.l, s13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v66.l, v39.l, s15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v67.l, v48.l, s17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v39
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v98.l, v99.l, v54.l, s17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v114.l, v131.l, v66.l, s25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v33
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v34
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 16, v24
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 16, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 16, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v117, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v134, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v30
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v82, v82
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v145, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v118, v118
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v87.l, v87.l, v53.l, s15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v54
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v100.l, v103.l, v55.l, s19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v118.l, v147.l, v48.l, s29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v98
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s17, v69, v130
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s18, v80, v131
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s19, v81, v132
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 16, v26
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 16, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v85, v85
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v117, v117
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v134, v134
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v52, v52
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v64.l, v38.l, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v84.l, v83.l, s12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v116.l, v115.l, s20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v145, v145
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v53
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v87
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v118
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v97, v146
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v35.l, s17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v37.l, s18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v39.l, s19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v129, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v65, v65
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v28
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v50, v50
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v68.l, v67.l, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v144.l, v135.l, s26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v71.l, v51.l, s11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v83.l, v52.l, s13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v102.l, v115.l, v64.l, s21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v96, v145
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v117, v81
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v81.l, v98.l, v54.l, s25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v33
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v34
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v36
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v129, v129
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v65, v65
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v128.l, v119.l, s22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v51
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v52
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v64
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v116.l, v135.l, v68.l, s27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v71
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v83
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v102
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v35.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v96
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v97
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v98
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v112.l, v119.l, v65.l, s23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v66
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v114
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v85, v135
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v86, v144
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v101, v69
+; GFX11-TRUE16-NEXT: s_and_b32 s17, s17, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v37.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v39.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v38.l, v49.l, s7
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v113, v80
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v71.l, v51.l, s22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v83.l, v52.l, s23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v102.l, v64.l, s27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s43
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s45
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v33.l, v35.l, s17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v38
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v114.l, v66.l, s29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v83
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s18, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s19, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v34.l, v37.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v36.l, v39.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v50.l, s9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v64.l
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v82, v133
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v114
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v50
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v55
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v67
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v100
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v38.l, v49.l, s20
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s27, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v84, v134
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v99, v147
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v38
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v71
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v29
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v50.l, s21
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v99
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s46
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v52.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v65
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v112
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v87.l, v53.l, s24
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v102
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s47
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v128, v82
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v100.l, v55.l, s26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v67
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s20, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v28
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v38.l, v49.l, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v103, v130
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v80
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s23, s6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v50.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s56
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v100
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.h, v71.l, v52.l, s6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v53.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v103
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v30
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s21, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v67.l, v50.l, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s24, s7
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v119, v132
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v80.l, v53.l, s7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v29
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v13.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s43
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v68
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v128
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v116
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v84.l, v112.l, v65.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v30
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v69
+; GFX11-TRUE16-NEXT: s_and_b32 s16, s44, s16
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v115, v131
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v119
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v81
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v87.l, v118.l, v48.l, s42
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v86.l, v116.l, v68.l, s41
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v51.l
+; GFX11-TRUE16-NEXT: s_and_b32 s15, s43, s15
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v54.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s15
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v101
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v112
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v82
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v84
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v85
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v86
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v87
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s22, s5
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s25, s8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v55.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v65.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v66.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v68.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v48.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v113
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v115
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v116
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v117
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v118
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v69.l, v51.l, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v81.l, v54.l, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s26, s9
+; GFX11-TRUE16-NEXT: s_and_b32 s11, s28, s11
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s29, s12
+; GFX11-TRUE16-NEXT: s_and_b32 s13, s41, s13
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s42, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v82.l, v55.l, s9
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v31
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v15.h, v31.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 16, v31
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v97, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s40
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v70, v70
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v31.h, v32.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v32
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v32.l, v96.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v97, v97
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v98, v98
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v35, 16, v32
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v96.l, v32.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v84.l, v65.l, s11
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v30
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v31, v34
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v33, v51
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v29.l, v32.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v30.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v32.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v26.l, v32.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v27.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s57, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v33
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.h, v83.l, v64.l, s10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v35, v37
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v34, v39
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v32.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v15.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v33
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v35, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v85.l, v66.l, s12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v34
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v35
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v86.l, v68.l, s13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v87.l, v48.l, s14
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v70, v70
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v33.l, v32.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v31.l, v15.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v129, v133
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v38, v38
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v37, v39
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v39, v38
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v27
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v11.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v39, v38
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v7
+; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v38, v38
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v39, v39
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v25.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v23.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v38, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v113.l, v50.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v50.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s56, s19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v68.l, v49.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v26, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v5
; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v28, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v24.l, v50.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v2.h, v18.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v26, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v22.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v27.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v21.l, v22.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v25, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v27.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v26, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v24.l, v8.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v9.l, v7.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v27.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v17.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v17.h, v24.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v27.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v20.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v22.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v21.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v20, v20
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v23, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v0.h, v16.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v16.h, v23.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v24.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v26
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v29, v28
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v31, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v22.l, v23.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v24.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v16
-; GFX11-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v26
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v23.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v27
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v25.l, v24.l, s9
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.l, v23.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v0.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v17.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v19.l, v3.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v20.l, v4.l, s7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v18.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v17.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v19.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v21, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v18
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v5.l, v0.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v9.l, v1.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v6.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v7.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v8.l, v4.l, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v35 :: v_dual_mov_b32 v1, v34
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v33 :: v_dual_mov_b32 v3, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v31 :: v_dual_mov_b32 v5, v30
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v29 :: v_dual_mov_b32 v7, v28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v27 :: v_dual_mov_b32 v9, v36
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v32bf16:
@@ -11031,544 +11201,589 @@ define <32 x bfloat> @v_maximumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v30
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v34, v34
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v35, v35
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v48, v48
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v36, v36
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v30
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v30
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v29
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v13
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v29
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v28
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v12
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v28
; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v49, v49
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v50, v50
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v51, v51
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v54, v54
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v68, v68
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s7
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v30.h, v32.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v37, v37
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v52, v52
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v55, v55
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v66, v66
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v69, v69
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s59, v114, v114
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v32
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v51
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v130, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v51, v51
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v53, v53
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v26
; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v115, 0xffff0000, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v64, v64
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 16, v22
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v101, 0xffff0000, v22
; GFX12-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v38, v38
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v65, v65
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v67, v67
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v80, v80
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 16, v18
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v133, 0xffff0000, v18
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v146, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v52, v52
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v54, v54
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v29
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v66, v66
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v98, v98
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v130, v130
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v27.h, v35.l, s14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v29
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v52, v114
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v53, v53
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v70, v70
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s60, v115, v115
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v30
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v103, 0xffff0000, v21
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v33, v33
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v39, v39
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v102, v102
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v24
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v71, v71
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v83, v83
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v84, v84
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s23
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v34
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v35.l, v33.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v37.l, v34.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v39.l, v36.l, s4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 16, v25
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 16, v9
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v25
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 16, v23
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v97, 0xffff0000, v23
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 16, v21
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 16, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v113, 0xffff0000, v21
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v55, v55
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v69, v69
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v27
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v70, v70
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v86, v86
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v101, v101
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v102, v102
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v133, v133
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v146, v146
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v54, v54
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v66, v66
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v100.l, v99.l, s16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v132.l, v131.l, s24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v35.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v37.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v39.l, s5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v118, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v81, v81
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v97, v97
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v113, v113
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v114, v114
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v51, v51
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v53, v53
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v55, v55
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v69, v69
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v80.l, v71.l, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v96.l, v87.l, s14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v112.l, v103.l, s18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v48.l, v147.l, s28
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v35
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v26.h, v36.l, s18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v65
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v68
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v64, v117
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v39
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v98.l, v99.l, v54.l, s17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v114.l, v131.l, v66.l, s25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v33
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v34
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v36
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v27
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 16, v11
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 16, v24
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 16, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v24
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 16, v20
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 16, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v117, 0xffff0000, v20
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v134, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v30
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v82, v82
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v145, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v118, v118
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v87.l, v87.l, v53.l, s15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v100.l, v103.l, v55.l, s19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v118.l, v147.l, v48.l, s29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v98
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s17, v69, v130
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s18, v80, v131
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s19, v81, v132
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v27
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 16, v26
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 16, v10
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 16, v17
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v15
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v85, v85
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v117, v117
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v134, v134
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v52, v52
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v64.l, v38.l, s6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v32.l, s20
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s21
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s16
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v53, v115
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v103, v103
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s47
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v29.h, v33.l, s6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v85, v85
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v36
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s24
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v24.h, v38.l, s26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v71
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v66, v118
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v69, v128
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v51
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v84.l, v83.l, s12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v116.l, v115.l, s20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v145, v145
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v53
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v87
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v118
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v97, v146
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v35.l, s17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v37.l, s18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v39.l, s19
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 16, v19
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 16, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v129, 0xffff0000, v19
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v65, v65
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v28
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v50, v50
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v68.l, v67.l, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v144.l, v135.l, s26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v71.l, v51.l, s11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v83.l, v52.l, s13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v102.l, v115.l, v64.l, s21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s24, v96, v145
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v117, v81
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v33
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v102.l, v21.h, v49.l, s56
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v54
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v86, v86
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v32.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v26
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v80, v130
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v65.l, v34.l, s24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v29
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v69
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v70, v129
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v30
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v25.h, v37.l, s22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v49
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v102
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v55, v116
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v87, v87
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v81.l, v98.l, v54.l, s25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v33
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v34
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v36
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v129, v129
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v65, v65
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v128.l, v119.l, s22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v52
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v64
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v116.l, v135.l, v68.l, s27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v71
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v83
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v102
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v35.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v96
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v97
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v98
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s29
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s41
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s45
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v13.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v24
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v81, v131
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v53
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v80
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s20, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s57, v112, v112
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v70
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v51.l, v32.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s56, v103, v52
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.l, v33.l, s22
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s43
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v14.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v34.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v38
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s42
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s46
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v85
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v87, v135
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s29
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v81
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s23, s4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s58, v113, v113
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s57
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v112.l, v119.l, v65.l, s23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v66
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v114
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s22, v85, v135
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s23, v86, v144
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s27, v101, v69
+; GFX12-TRUE16-NEXT: s_and_b32 s17, s17, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v37.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v39.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v38.l, v49.l, s7
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s29, v113, v80
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v71.l, v51.l, s22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v83.l, v52.l, s23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v102.l, v64.l, s27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s43
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s45
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v96.l, v23.h, v39.l, s40
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v25
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v71.l, v36.l, s28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v52
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s42, v86, v134
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s43
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v26
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s21, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s24, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v12
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s58
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v33.l, v35.l, s17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v38
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v114.l, v66.l, s29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v83
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s18, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s19, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v53.l, v34.l, s5
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v33.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v39
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v96
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v84, v133
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s27
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v71
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v10.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v99.l, v22.h, v48.l, s44
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s45, v98, v145
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s47, v101, v147
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v24
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v86
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v97, v144
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v34.l, v37.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v36.l, v39.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v50.l, s9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0, v64.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s20, v82, v133
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v114
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v50
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v55
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v67
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v100
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v38.l, v49.l, s20
+; GFX12-TRUE16-NEXT: s_and_b32 s10, s27, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s21, v84, v134
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s26, v99, v147
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v38
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v71
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v49.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v29
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s41
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v27
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s22, s3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v48
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v99
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s45
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s47
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v98
-; GFX12-TRUE16-NEXT: s_and_b32 s10, s29, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v50.l, s21
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v99
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s46
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v52.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v65
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v112
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v87.l, v53.l, s24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v102
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s47
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s44, v128, v82
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v100.l, v55.l, s26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v67
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s20, s3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v28
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v52.l, v33.l, s3
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s57, v112, v114
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s10
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v11.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v96.l, v39.l, s44
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v84
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s46, v100, v146
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v68.l, v35.l, s26
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v102.l, v49.l, s56
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v22
-; GFX12-TRUE16-NEXT: s_and_b32 s14, s43, s14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s59
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s57
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v38.l, v49.l, s3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s28, v103, v130
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v80
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s23, s6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v50.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s56
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v100
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s12
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v9.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v96
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s27, s8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v7.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v100
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s47, 0, v102
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.h, v71.l, v52.l, s6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v53.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v103
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v30
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s21, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v113.l, v20.h, v50.l, s60
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v21
-; GFX12-TRUE16-NEXT: s_and_b32 s12, s41, s12
-; GFX12-TRUE16-NEXT: s_and_b32 s16, s45, s16
-; GFX12-TRUE16-NEXT: s_and_b32 s18, s47, s18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v67.l, v50.l, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s24, s7
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s43, v119, v132
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s12
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s57, 0, v112
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s16
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v113
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v50
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v37
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v82
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v68
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v12.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s25, v67, v119
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0, v49.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s40, v83, v132
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s56, 0, v103
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v80.l, v53.l, s7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v29
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0, v13.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v14.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v35.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v82.l, v37.l, s40
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v85.l, v38.l, s42
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v83
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v55
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v99.l, v48.l, s46
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v36.l
-; GFX12-TRUE16-NEXT: s_and_b32 s7, s26, s7
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v82
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s43
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v68
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v128
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v116
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v84.l, v112.l, v65.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v30
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v69
+; GFX12-TRUE16-NEXT: s_and_b32 s16, s44, s16
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s41, v115, v131
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v54.l, v35.l, s7
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v85
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v64
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v65
-; GFX12-TRUE16-NEXT: s_and_b32 s6, s25, s6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v66
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v119
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v81
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v87.l, v118.l, v48.l, s42
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v86.l, v116.l, v68.l, s41
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0, v51.l
+; GFX12-TRUE16-NEXT: s_and_b32 s15, s43, s15
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v54.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v67
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s28, s9
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v37.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v38.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0, v39.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0, v48.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v87
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v97
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v99
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s46, 0, v101
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s15
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v101
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v112
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v82
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v84
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v85
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v86
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v87
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s22, s5
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s25, s8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0, v55.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0, v65.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0, v66.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0, v68.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0, v48.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v113
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v115
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v116
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v117
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v118
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v55.l, v36.l, s9
-; GFX12-TRUE16-NEXT: s_and_b32 s11, s40, s11
-; GFX12-TRUE16-NEXT: s_and_b32 s13, s42, s13
-; GFX12-TRUE16-NEXT: s_and_b32 s15, s44, s15
-; GFX12-TRUE16-NEXT: s_and_b32 s17, s46, s17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v69.l, v51.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v81.l, v54.l, s8
+; GFX12-TRUE16-NEXT: s_and_b32 s9, s26, s9
+; GFX12-TRUE16-NEXT: s_and_b32 s11, s28, s11
+; GFX12-TRUE16-NEXT: s_and_b32 s12, s29, s12
+; GFX12-TRUE16-NEXT: s_and_b32 s13, s41, s13
+; GFX12-TRUE16-NEXT: s_and_b32 s14, s42, s14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v64.l, v37.l, s11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v65.l, v38.l, s13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v66.l, v39.l, s15
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v67.l, v48.l, s17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v82.l, v55.l, s9
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v31
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v31
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 16, v31
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v97, 0xffff0000, v31
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v31
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s40
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v15.h, v31.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v70, v70
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v32.l, v96.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v97, v97
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v98, v98
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v15
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v35, 16, v32
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v31.h, v32.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v96.l, v32.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v32
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v84.l, v65.l, s11
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v30
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v31, v34
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v33, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v33
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v31
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.h, v83.l, v64.l, s10
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v35, v37
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v34, v39
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v29.l, v32.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v32.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v30.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v32.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v15.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v5.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v15.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v32.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v33
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v35, 16, v31
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v85.l, v66.l, s12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v34
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v35
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v86.l, v68.l, s13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v87.l, v48.l, s14
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v70, v70
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v26.l, v32.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v27.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s57, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v33.l, v32.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v31.l, v15.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v129, v133
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v113.l, v50.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, s3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v38, v38
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v37, v39
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v26
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v50.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s56, s19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v68.l, v49.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v26
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v28, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v12.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v39, v38
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v27
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v11.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s3, v39, v38
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v26
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v24.l, v50.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v18
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s0, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v7
+; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v38, v38
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v39, v39
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v25.l, v9.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v2.h, v18.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v26, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v22.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v23.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v38, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v27.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v21.l, v22.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v25, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v26, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v7.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v26, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v24.l, v8.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v9.l, v7.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v27.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v17.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v17.h, v24.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v19
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v20.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v5.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v27.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v22.l, v6.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s1, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v23, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v9, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v21.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v20, v20
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v0.h, v16.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v16.h, v23.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v23
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v18.l, v2.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v24.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v27, v26
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v29, v28
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v31, v30
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v17.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v7, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v19.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v18, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v21, v20
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v22.l, v23.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v24.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v16
-; GFX12-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v26
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v27
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v25.l, v24.l, s9
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.l, v23.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v18
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s5, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v0.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v17.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v19.l, v3.l, s8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v20.l, v4.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v5.l, v0.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v9.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v6.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v7.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v8.l, v4.l, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v35 :: v_dual_mov_b32 v1, v34
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v33 :: v_dual_mov_b32 v3, v32
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v31 :: v_dual_mov_b32 v5, v30
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v29 :: v_dual_mov_b32 v7, v28
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v27 :: v_dual_mov_b32 v9, v36
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v32bf16:
@@ -12327,26 +12542,27 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -12381,17 +12597,19 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
@@ -12613,19 +12831,20 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v3.l, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
@@ -12703,21 +12922,22 @@ define <2 x bfloat> @v_maximumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
@@ -13042,58 +13262,58 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX11-TRUE16-LABEL: v_maximumnum_v3bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v9
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v9, v8
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v6.l, s2
; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee:
@@ -13161,62 +13381,65 @@ define <3 x bfloat> @v_maximumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v5, v9
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v7, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v8, v11
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v9, v8
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v10, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v6.l, s2
; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v3bf16_no_ieee:
@@ -13611,65 +13834,69 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.l, s5
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee:
@@ -13761,74 +13988,81 @@ define <4 x bfloat> @v_maximumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s2, v11, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v8, v9
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, v11, v10
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0, v0.l
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e64 s1, v13, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v4.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v6.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0, v1.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v4.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.l, s5
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_maximumnum_v4bf16_no_ieee:
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index 7f6ae93864ebb..7615f660c98b5 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -109,26 +109,27 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -163,17 +164,19 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
@@ -295,8 +298,11 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
@@ -331,8 +337,11 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
@@ -552,19 +561,20 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v3.l, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
@@ -642,21 +652,22 @@ define <2 x bfloat> @v_minimumnum_v2bf16(<2 x bfloat> %x, <2 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
@@ -868,24 +879,26 @@ define <2 x bfloat> @v_minimumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.h, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v2bf16_nnan:
@@ -928,26 +941,28 @@ define <2 x bfloat> @v_minimumnum_v2bf16_nnan(<2 x bfloat> %x, <2 x bfloat> %y)
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v1.h, v0.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v0.h, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v2.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v2bf16_nnan:
@@ -1238,58 +1253,58 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) #1 {
; GFX11-TRUE16-LABEL: v_minimumnum_v3bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v9
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v9, v8
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v6.l, s2
; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16:
@@ -1357,62 +1372,65 @@ define <3 x bfloat> @v_minimumnum_v3bf16(<3 x bfloat> %x, <3 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v9
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v11
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v9, v8
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v6.l, s2
; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16:
@@ -1664,30 +1682,32 @@ define <3 x bfloat> @v_minimumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s4, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16_nnan:
@@ -1741,33 +1761,33 @@ define <3 x bfloat> @v_minimumnum_v3bf16_nnan(<3 x bfloat> %x, <3 x bfloat> %y)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s4, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16_nnan:
@@ -2148,65 +2168,69 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) #1 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.l, s5
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16:
@@ -2298,74 +2322,81 @@ define <4 x bfloat> @v_minimumnum_v4bf16(<4 x bfloat> %x, <4 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v4.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.l, s5
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16:
@@ -2690,40 +2721,44 @@ define <4 x bfloat> @v_minimumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v1.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s0, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.h, s5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v8
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v6.l, s5
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16_nnan:
@@ -2788,45 +2823,49 @@ define <4 x bfloat> @v_minimumnum_v4bf16_nnan(<4 x bfloat> %x, <4 x bfloat> %y)
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v6
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v9, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v3.h, v1.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v1.h
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v2.h, v0.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.h
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v8
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v5, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s0, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v5
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v1.h, s5
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v0.h, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v7.l, v6.l, s5
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16_nnan:
@@ -3376,95 +3415,101 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) #1 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v5.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v5.h, v7.l, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v11, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.h, v4.h, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v8, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v12, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v7.l, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v6.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v9, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v7.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v9.l, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.l, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v14.l, v13.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v8.l, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v6, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v14, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v10.l, v9.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v8.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v18, v19
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v13.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v11
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v14, v13
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v16, v15
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v11
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s0, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v6.l, s7
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v9.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v9.l, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v12.l, s6
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v6bf16:
@@ -3593,107 +3638,115 @@ define <6 x bfloat> @v_minimumnum_v6bf16(<6 x bfloat> %x, <6 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v2.h, v5.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v5.h, v7.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v10, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v11, v11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v1.h, v4.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.l, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v8, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v12, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v14.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v16.l, v7.l, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v4.h, v6.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v14, v14
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s5, v9, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v8.l, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v6, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v14, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v5.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v4.l, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v0.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v10.l, v9.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v8.l, v7.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v3.h, v9.l, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v7.l, v8.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v17, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v6.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v12, v11
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v14, v13
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v16, v15
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v18, v19
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v13.l, v12.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v14, v13
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v16, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v8.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v1.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v11
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v9.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s0, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v10.l, v6.l, s7
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v9.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.l, v9.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v12.l, s6
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v3.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v4.l, v1.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v2.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v6bf16:
@@ -4475,126 +4528,139 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) #1 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.h, v7.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v10.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v19, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v15.l, v12.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v13.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v17
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v12.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v9.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.h, v6.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v15, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v13.l, s0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v14.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v13.l
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v15.l, v12.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v9.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v6.h, v8.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v12.l, v8.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v3.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v10.l, v13.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v15, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v14.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v8.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v14, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v16.l, v8.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v9.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v1.h, v5.h, s4
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v15, v14
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v8.l
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s0, s1
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v12.l, v8.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v0.h, v4.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v13.l, v10.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v11.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v13
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v11.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s0, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s8, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v11.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v12.l, v10.l, s8
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v11.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v8.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v17.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, s5
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v8bf16:
@@ -4753,146 +4819,161 @@ define <8 x bfloat> @v_minimumnum_v8bf16(<8 x bfloat> %x, <8 x bfloat> %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v7
; GFX12-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v9, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v10, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.h, v7.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v17, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v7.h, v9.l, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v19, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v2.h, v6.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v9.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v6.h, v8.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v19, v19
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v9.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v14, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v15.l, v12.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v13.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v12.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v16.l, v8.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v8.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v16
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v15, v17
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v18
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v13.l, s0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v14.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v13.l
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v9.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v15.l, v12.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v14.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v12.l, v8.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v17
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s0, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v3.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v10.l, v9.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v1.h, v5.h, s4
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v10.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v5.h, v10.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v8.l
-; GFX12-TRUE16-NEXT: s_and_b32 s7, s0, s1
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v10
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v15, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, v3.l, s7
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v12.l, v8.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v11.l, v14.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v8.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v13, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v11, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v17
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v0.h, v4.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v8, v8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v12, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v13.l, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v11, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v4.h, v11.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v13
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v11, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v3.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v13, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v15, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v11.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v8
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s0, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v5
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s8, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v9
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v11.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v0.l
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s3, s6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v12.l, v10.l, s8
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v11.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v8.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v17.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s4, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v1.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v6.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v4.l, v0.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v8
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v8bf16:
@@ -6310,253 +6391,276 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) #
; GFX11-TRUE16-LABEL: v_minimumnum_v16bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v7.h, v15.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v22
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v18.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v17.l, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v23.l, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v25
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v19.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s1, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v19.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v17.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v26.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v24, v25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v20.l, v21.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v22.l, v23.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.h, v18.l, v19.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v21.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v19, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v24.l, v20.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v18.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v23.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v19.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v17
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s0, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v26, v27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v19
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v18.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.h, v14.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v17.l, v18.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.h, v21.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v21.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v16.l, v18.l, s2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v14
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s3, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v20, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v5.h, v13.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v24.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v18.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v26, v27
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v28.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v22.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.h, v7.l, v21.l, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v26, v27
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s0, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v22.l
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v24.l, v28.l, s3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.h, v17.l, v23.l, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.l, v18.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v28.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v24.l, v21.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v19.l, v22.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v21.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v28.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v18.l, v17.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v23
; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v21.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v19.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v16.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v7
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v22
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v20, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v17.l, v21.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v22
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v15.l, v17.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v16.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v12.h, v17.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v19.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s1, s2
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v15
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v3.h, v11.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v19.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v15.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v20, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v21, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v17.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v15.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v21
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v15.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v19.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v19.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v13.l, v5.l, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v19.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v1.h, v9.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v9.h, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v19.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v0.h, v8.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v8.h, v15.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v10.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v9.l, v1.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v19, v18
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v14.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v8
-; GFX11-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v18
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v15.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v19
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v17.l, v16.l, s9
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v13.l, v15.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.l, v3.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v12.l, v4.l, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v12.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v10
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v21, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v5.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v8.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v13
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v1.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v11.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v9.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v10.l, v4.l, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v19 :: v_dual_mov_b32 v5, v18
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v17
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v16bf16:
@@ -6882,294 +6986,320 @@ define <16 x bfloat> @v_minimumnum_v16bf16(<16 x bfloat> %x, <16 x bfloat> %y) #
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v17, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v15
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v27, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v15
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v29, 0xffff0000, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v7.h, v15.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v15
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v17
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v17.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v18.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v5
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v17.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v15.h, v18.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v18.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v23.l, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v25
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v19.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s1, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v19.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v7.l, v17.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v26.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v24, v25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v27, v27
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v3
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v20.l, v21.l, s2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v22.l, v23.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v20, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.h, v18.l, v19.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v26, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v20, v20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v18
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v21.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v19, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v7.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v24.l, v20.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v10
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v18
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v6.h, v14.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v18.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v17.l, v18.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v23.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v19.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v24, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v17
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s0, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.h, v21.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v20, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v17
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v7.l, s0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v15
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v21.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v19
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v16.l, v18.l, s2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v17
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v14
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v13
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s3, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v20, v19
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v5.h, v13.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v26, v27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v19
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v18.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v24.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v29, v29
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v18, v18
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v21.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v13.h, v19.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v18.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v26, v27
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v28
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v28.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v22.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v23.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v7.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s3
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v24, v24
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v20, v18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.h, v7.l, v21.l, s3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v26, v27
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s0, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v25
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v22.l
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v24.l, v28.l, s3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v17.l, v21.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v22
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.h, v17.l, v23.l, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.h, v20.l, v18.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v28.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v4.h, v12.h, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v15, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v5.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v24.l, v21.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v25, v25
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v26, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v15
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v14.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v12.h, v17.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v15
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v19.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s1, s2
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v15
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v19.l, v22.l, s3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v21.l, v17.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v3.h, v11.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v15.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v18
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v16.l, v19.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v13.l, v5.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v11.h, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v7.l, v28.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v20, v19
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v16, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v21, v18
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v14.l, v17.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v15.l, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v22, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v15.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v14.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v18.l, v17.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v2.h, v10.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v12
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v18, v17
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v15.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v13.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v10.h, v19.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v16.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v17.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v23, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v13.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v17, v14
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v16.l, v19.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v15.l, v17.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v14
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v16.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v15, v15
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v19.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v12.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v1.h, v9.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v21
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v11.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v5.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v9.h, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v19.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v18, v18
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v15, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s1, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v9
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v15, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v14.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v21, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v22, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v10
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v13.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v10.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v22, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v0.h, v8.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v13, v13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v9.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v14, v14
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v8.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v14, v14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v21, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v10.l, v2.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v8.h, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v9.l, v1.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v16.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v19, v18
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v21, v20
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v13
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v12.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v12, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v11.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v14, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v10
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v21, v16
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v14.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v16.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v9
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v13
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v19, 16, v8
-; GFX12-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v14
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v18
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v15.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v19
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v17.l, v16.l, s9
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v13.l, v15.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v5.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v8.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v13
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s5, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v8.l, v0.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v9.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v10.l, v2.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v11.l, v3.l, s8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v12.l, v4.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v11.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v9.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v10.l, v4.l, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v5 :: v_dual_mov_b32 v1, v6
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v15 :: v_dual_mov_b32 v3, v20
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v19 :: v_dual_mov_b32 v5, v18
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v17
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v16bf16:
@@ -9998,494 +10128,534 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v34, v34
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v35, v35
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v48, v48
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v36, v36
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v13
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v30
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v14
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v30
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v29
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v13
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v29
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v28
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v28
; GFX11-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v49, v49
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v50, v50
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v51, v51
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v54, v54
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v68, v68
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v30.h, v32.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v37, v37
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v52, v52
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v55, v55
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v66, v66
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v69, v69
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s59, v114, v114
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v32
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v51
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v130, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v51, v51
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v53, v53
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v26
; GFX11-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v115, 0xffff0000, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v64, v64
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 16, v22
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v101, 0xffff0000, v22
; GFX11-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v38, v38
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v65, v65
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v67, v67
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v80, v80
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v27.h, v35.l, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s20
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v29
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v52, v114
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v53, v53
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v70, v70
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s60, v115, v115
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s15
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v14
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v30
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v103, 0xffff0000, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v33, v33
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v39, v39
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v102, v102
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v24
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v71, v71
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v83, v83
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v84, v84
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v133, 0xffff0000, v18
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v146, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v52, v52
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v54, v54
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v29
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v66, v66
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v98, v98
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v130, v130
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v35.l, v33.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v37.l, v34.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v39.l, v36.l, s4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 16, v25
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 16, v9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v25
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 16, v23
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 16, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v97, 0xffff0000, v23
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 16, v21
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 16, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v113, 0xffff0000, v21
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 16, v16
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v14
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v13
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v55, v55
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v69, v69
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v27
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v70, v70
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v86, v86
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v101, v101
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v102, v102
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v133, v133
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v146, v146
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v54, v54
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v66, v66
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v100.l, v99.l, s16
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v132.l, v131.l, s24
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v35.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v37.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v39.l, s5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v118, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v81, v81
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v97, v97
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v113, v113
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v114, v114
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v49, v49
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v51, v51
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v53, v53
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v55, v55
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v69, v69
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v80.l, v71.l, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v96.l, v87.l, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v112.l, v103.l, s18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v48.l, v147.l, s28
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v35
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v26.h, v36.l, s18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v65
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v68
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v64, v117
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v32.l, s20
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s16
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v53, v115
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v103, v103
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s47
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v29.h, v33.l, s6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v85, v85
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v36
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v24.h, v38.l, s26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v71
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v66, v118
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v69, v128
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v51
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v102.l, v21.h, v49.l, s56
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v54
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v86, v86
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v32.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v26
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v80, v130
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v53.l, v65.l, v34.l, s24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v29
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v69
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v70, v129
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v30
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v25.h, v37.l, s22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v49
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v102
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v55, v116
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v87, v87
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s29
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s41
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s45
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v13.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v8
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v24
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v81, v131
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v53
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v80
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s20, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s57, v112, v112
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v70
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v51.l, v32.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s56, v103, v52
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.l, v33.l, s22
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s43
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v14.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v34.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v38
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s42
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s46
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v85
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v87, v135
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s29
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v81
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s23, s4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s58, v113, v113
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s57
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v96.l, v23.h, v39.l, s40
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v55.l, v71.l, v36.l, s28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v52
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v22
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v86, v134
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s43
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v26
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s21, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s24, s5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s58
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v53.l, v34.l, s5
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v33.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v39
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v96
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v84, v133
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s27
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v71
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v10.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v99.l, v22.h, v48.l, s44
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s45, v98, v145
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s47, v101, v147
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v24
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v86
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v97, v144
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s41
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v27
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s22, s3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v8.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v48
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v99
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s45
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s47
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v98
-; GFX11-TRUE16-NEXT: s_and_b32 s10, s29, s10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v52.l, v33.l, s3
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s57, v112, v114
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s10
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v11.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v66.l, v96.l, v39.l, s44
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v84
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s46, v100, v146
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v54.l, v68.l, v35.l, s26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v102.l, v49.l, s56
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v22
-; GFX11-TRUE16-NEXT: s_and_b32 s14, s43, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s59
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s57
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s12
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v9.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v96
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s27, s8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v7.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v6.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v100
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s47, 0, v102
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v113.l, v20.h, v50.l, s60
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v21
-; GFX11-TRUE16-NEXT: s_and_b32 s12, s41, s12
-; GFX11-TRUE16-NEXT: s_and_b32 s16, s45, s16
-; GFX11-TRUE16-NEXT: s_and_b32 s18, s47, s18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s12
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s57, 0, v112
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v113
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v50
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v37
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v82
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v68
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v12.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v67, v119
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v49.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v83, v132
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s56, 0, v103
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v54
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v35.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v82.l, v37.l, s40
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v85.l, v38.l, s42
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v83
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v55
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v99.l, v48.l, s46
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v36.l
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s26, s7
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v82
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v54.l, v35.l, s7
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v85
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v64
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v65
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s25, s6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v66
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v67
-; GFX11-TRUE16-NEXT: s_and_b32 s9, s28, s9
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v37.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v38.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v39.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v48.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v87
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v97
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v99
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s46, 0, v101
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v55.l, v36.l, s9
-; GFX11-TRUE16-NEXT: s_and_b32 s11, s40, s11
-; GFX11-TRUE16-NEXT: s_and_b32 s13, s42, s13
-; GFX11-TRUE16-NEXT: s_and_b32 s15, s44, s15
-; GFX11-TRUE16-NEXT: s_and_b32 s17, s46, s17
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.h, v64.l, v37.l, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.h, v65.l, v38.l, s13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v66.l, v39.l, s15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v67.l, v48.l, s17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v39
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v98.l, v99.l, v54.l, s17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v114.l, v131.l, v66.l, s25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v33
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v34
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 16, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 16, v24
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 16, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v24
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 16, v20
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v117, 0xffff0000, v20
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v134, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v30
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v82, v82
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v145, 0xffff0000, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v118, v118
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v87.l, v87.l, v53.l, s15
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v54
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v100.l, v103.l, v55.l, s19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v118.l, v147.l, v48.l, s29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v98
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s17, v69, v130
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s18, v80, v131
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s19, v81, v132
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v27
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 16, v26
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 16, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v85, v85
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v117, v117
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v134, v134
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v52, v52
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v49.l, v64.l, v38.l, s6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v52.l, v84.l, v83.l, s12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v64.l, v116.l, v115.l, s20
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v145, v145
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v53
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v48
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v87
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v118
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v97, v146
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v35.l, s17
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v37.l, s18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v39.l, s19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v129, 0xffff0000, v19
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v65, v65
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v28
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v50, v50
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v50.l, v68.l, v67.l, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v68.l, v144.l, v135.l, s26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v71.l, v51.l, s11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v83.l, v52.l, s13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v102.l, v115.l, v64.l, s21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v96, v145
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v117, v81
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v81.l, v98.l, v54.l, s25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v33
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v34
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v36
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v129, v129
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v65, v65
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v65.l, v128.l, v119.l, s22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v51
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v52
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v64
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v116.l, v135.l, v68.l, s27
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v71
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v83
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v102
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v15
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v35.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v96
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v97
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v98
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v112.l, v119.l, v65.l, s23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v66
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v114
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v85, v135
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v86, v144
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v101, v69
+; GFX11-TRUE16-NEXT: s_and_b32 s17, s17, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v37.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v39.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v38.l, v49.l, s7
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v113, v80
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v69.l, v71.l, v51.l, s22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v71.l, v83.l, v52.l, s23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v83.l, v102.l, v64.l, s27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s43
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s45
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.h, v33.l, v35.l, s17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v49
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v38
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v85.l, v114.l, v66.l, s29
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v83
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s18, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s19, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v13
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.h, v34.l, v37.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.h, v36.l, v39.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v50.l, s9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v64.l
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v82, v133
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v114
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v50
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v55
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v67
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v100
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v38.l, v38.l, v49.l, s20
+; GFX11-TRUE16-NEXT: s_and_b32 s10, s27, s10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v84, v134
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v99, v147
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v38
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v71
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v29
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v50.l, s21
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v99
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s46
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v52.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v65
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v112
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v80.l, v87.l, v53.l, s24
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v102
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s47
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v128, v82
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v82.l, v100.l, v55.l, s26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v67
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s20, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v28
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.h, v38.l, v49.l, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v103, v130
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v80
+; GFX11-TRUE16-NEXT: s_and_b32 s6, s23, s6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v50.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s56
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v100
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.h, v71.l, v52.l, s6
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v53.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v103
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v30
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s21, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.h, v67.l, v50.l, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s7, s24, s7
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v119, v132
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.h, v80.l, v53.l, s7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v29
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v13.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s43
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v68
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v128
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v116
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v84.l, v112.l, v65.l, s28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v30
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v69
+; GFX11-TRUE16-NEXT: s_and_b32 s16, s44, s16
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v115, v131
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v119
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v81
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v87.l, v118.l, v48.l, s42
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v86.l, v116.l, v68.l, s41
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v51.l
+; GFX11-TRUE16-NEXT: s_and_b32 s15, s43, s15
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v54.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s15
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v101
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v112
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v82
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v84
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v85
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v86
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v87
+; GFX11-TRUE16-NEXT: s_and_b32 s5, s22, s5
+; GFX11-TRUE16-NEXT: s_and_b32 s8, s25, s8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v55.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v65.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v66.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v68.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v48.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v113
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v115
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v116
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v117
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v118
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.h, v69.l, v51.l, s5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.h, v81.l, v54.l, s8
+; GFX11-TRUE16-NEXT: s_and_b32 s9, s26, s9
+; GFX11-TRUE16-NEXT: s_and_b32 s11, s28, s11
+; GFX11-TRUE16-NEXT: s_and_b32 s12, s29, s12
+; GFX11-TRUE16-NEXT: s_and_b32 s13, s41, s13
+; GFX11-TRUE16-NEXT: s_and_b32 s14, s42, s14
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.h, v82.l, v55.l, s9
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v31
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v31
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v15.h, v31.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 16, v31
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v97, 0xffff0000, v31
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s40
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v70, v70
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v31.h, v32.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v32
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v32.l, v96.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v97, v97
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v98, v98
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v35, 16, v32
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v96.l, v32.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.h, v84.l, v65.l, s11
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v30
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v31, v34
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v33, v51
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v29.l, v32.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v30.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v32.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v15.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v26.l, v32.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v27.l, v15.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s57, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v33
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.h, v83.l, v64.l, s10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v35, v37
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v34, v39
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v27
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v32.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v33
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v35, 16, v31
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.h, v85.l, v66.l, s12
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v34
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v35
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.h, v86.l, v68.l, s13
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.h, v87.l, v48.l, s14
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v70, v70
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.h, v33.l, v32.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v15.l, v31.l, v15.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v129, v133
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v38, v38
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v37, v39
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v28
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v39, v38
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v27
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v11.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v39, v38
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s3
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, s2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v25
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v7
+; GFX11-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v38, v38
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v9.l
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v39, v39
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v23
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v24
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v36.l, v25.l, v9.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v23.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v22
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v38, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v113.l, v50.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, s3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v50.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v7.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s0
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v3
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s56, s19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v68.l, v49.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v26, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v5
; GFX11-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v28, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v24.l, v50.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v2.h, v18.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v26, v25
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v22.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v27.l, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v21.l, v22.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v25, v23
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v27.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v26, v25
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v27.l, v24.l, v8.l, s2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v28.l, v9.l, v7.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v27.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v17.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v17.h, v24.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v27.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s1, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v20
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v21
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v20.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v29.l, v22.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s2, s4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v30.l, v21.l, v5.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v17
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v19
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v20, v20
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v17
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v23, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v23.l, v0.h, v16.h, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v22.l, v16.h, v23.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v23
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v22
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v24.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v26
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v29, v28
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v31, v30
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v25
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v21.l, v22.l, v23.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v24.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v16
-; GFX11-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v26
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v23.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v27
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v25.l, v24.l, s9
-; GFX11-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.l, v23.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v0.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v17.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v19.l, v3.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v20.l, v4.l, s7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v18.l, v2.l, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v17.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v19.l, v3.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v21, v20
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v22
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v17
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v2.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v16
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v18
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s5, s8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v35.l, v5.l, v0.l, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v34.l, v9.l, v1.l, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v33.l, v6.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v32.l, v7.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v31.l, v8.l, v4.l, s2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v35 :: v_dual_mov_b32 v1, v34
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v33 :: v_dual_mov_b32 v3, v32
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v31 :: v_dual_mov_b32 v5, v30
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v29 :: v_dual_mov_b32 v7, v28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, v27 :: v_dual_mov_b32 v9, v36
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v32bf16:
@@ -11064,544 +11234,589 @@ define <32 x bfloat> @v_minimumnum_v32bf16(<32 x bfloat> %x, <32 x bfloat> %y) #
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: scratch_load_b32 v31, off, s32
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v34, 0xffff0000, v14
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v32, 0xffff0000, v15
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v35, 0xffff0000, v30
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v48, 16, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v36, 16, v14
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v34, v34
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v49, 16, v29
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v12
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v28
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v68, 16, v10
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v32, v32
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v35, v35
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v48, v48
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v14.h, v30.h, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v30
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v36, v36
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v38, 0xffff0000, v13
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v12
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v27
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v51, 0xffff0000, v13
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v53, 0xffff0000, v12
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v30
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v14
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v50, 0xffff0000, v30
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v29
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v13
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v52, 0xffff0000, v29
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v28
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v12
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v54, 0xffff0000, v28
; GFX12-TRUE16-NEXT: v_and_b32_e32 v66, 0xffff0000, v10
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v26
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v49, v49
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v50, v50
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v51, v51
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v54, v54
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v68, v68
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s7
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v30.h, v32.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v11
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v37, v37
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v39, 0xffff0000, v29
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v27
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v67, 0xffff0000, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v52, v52
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v55, v55
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v66, v66
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v69, v69
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s59, v114, v114
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v12.h, v28.h, s9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v11.h, v27.h, s13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, s19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v32
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v51
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v28
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v130, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v51, v51
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v53, v53
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v48, 0xffff0000, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v55, 0xffff0000, v11
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v26
; GFX12-TRUE16-NEXT: v_and_b32_e32 v70, 0xffff0000, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v115, 0xffff0000, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v64, v64
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v99, 16, v22
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v101, 0xffff0000, v22
; GFX12-TRUE16-NEXT: v_and_b32_e32 v102, 0xffff0000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v38, v38
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v71, 0xffff0000, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v25
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v83, 0xffff0000, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v65, v65
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v67, v67
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v80, v80
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v82, v82
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v131, 16, v18
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v132, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v133, 0xffff0000, v18
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v146, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v50, v50
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v52, v52
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s5, v54, v54
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v54, 16, v29
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s8, v66, v66
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s16, v98, v98
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v130, v130
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v10.h, v26.h, s17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v64, 16, v13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v28.h, v34.l, s10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v27.h, v35.l, s14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s20
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v29
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v52, v114
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v53, v53
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v70, v70
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s60, v115, v115
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s15
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v14
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v30
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v103, 0xffff0000, v21
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v33, v33
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v39, v39
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v102, v102
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v13.h, v29.h, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v24
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v71, v71
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s24, v81, v81
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v83, v83
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v84, v84
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s23
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v8.h, v24.h, s25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v66, 16, v34
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v35.l, v33.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v37.l, v34.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v39.l, v36.l, s4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v71, 16, v25
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v80, 16, v9
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v81, 0xffff0000, v25
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v87, 16, v23
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 16, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v97, 0xffff0000, v23
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v103, 16, v21
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v112, 16, v5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v113, 0xffff0000, v21
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v114, 0xffff0000, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v147, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v48, v48
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v48, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v49, 0xffff0000, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v14
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v53, 16, v13
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s6, v55, v55
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v12
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s9, v69, v69
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v27
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s10, v70, v70
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s14, v86, v86
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s17, v101, v101
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s18, v102, v102
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s25, v133, v133
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v146, v146
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v54, v54
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s47, v66, v66
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v100.l, v99.l, s16
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v132.l, v131.l, s24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v35.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v37.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v39.l, s5
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v82, 0xffff0000, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v118, 0xffff0000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s11, v81, v81
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s15, v97, v97
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s19, v113, v113
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s20, v114, v114
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v49, v49
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v51, v51
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v53, v53
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v55, v55
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v69, v69
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v80.l, v71.l, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v96.l, v87.l, s14
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v112.l, v103.l, s18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v48.l, v147.l, s28
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v35
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v26.h, v36.l, s18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v65
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v68
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v64, v117
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v39
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v98.l, v99.l, v54.l, s17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v114.l, v131.l, v66.l, s25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v33
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v34
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v36
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v27
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v64, 16, v11
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v83, 16, v24
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v84, 16, v8
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v85, 0xffff0000, v24
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v115, 16, v20
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v116, 16, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v117, 0xffff0000, v20
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v134, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v30
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s12, v82, v82
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v145, 0xffff0000, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s22, v118, v118
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v87.l, v87.l, v53.l, s15
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v100.l, v103.l, v55.l, s19
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v118.l, v147.l, v48.l, s29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v98
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s17, v69, v130
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s18, v80, v131
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s19, v81, v132
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v65, 0xffff0000, v27
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v67, 16, v26
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 16, v10
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v135, 16, v17
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v50, 16, v15
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s13, v85, v85
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s21, v117, v117
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s26, v134, v134
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v52, v52
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v64.l, v38.l, s6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v51.l, v51.l, v32.l, s20
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v37.l, v9.h, v25.h, s21
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s16
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v53, v115
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s56, v103, v103
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v49.l, v5.h, v21.h, s47
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v29.h, v33.l, s6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v86, 0xffff0000, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s28, v85, v85
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s27
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v36
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s24
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v24.h, v38.l, s26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v71
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v66, v118
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v69, v128
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v51
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v11
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v84.l, v83.l, s12
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v116.l, v115.l, s20
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s27, v145, v145
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v14.l, v30.l, s41
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v53
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v48
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v87
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v118
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v97, v146
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v35.l, s17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v34.l, v37.l, s18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v36.l, v39.l, s19
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v119, 16, v19
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v128, 16, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v129, 0xffff0000, v19
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s7, v65, v65
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v65, 16, v28
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v50, v50
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v68.l, v67.l, s8
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v144.l, v135.l, s26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v71.l, v51.l, s11
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v83.l, v52.l, s13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v102.l, v115.l, v64.l, s21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s42
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s24, v96, v145
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v117, v81
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v55, 16, v33
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v102.l, v21.h, v49.l, s56
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v54
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v87, 0xffff0000, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v98, 0xffff0000, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s29, v86, v86
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s41, v96, v96
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s45, v100, v100
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v32.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v26
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v80, v130
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v53.l, v65.l, v34.l, s24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v29
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v69
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v70, v129
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v30
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v25.h, v37.l, s22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v49
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v52, 16, v102
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v55, v116
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s40, v87, v87
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s42, v97, v97
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s43, v98, v98
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v101, v101
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v81.l, v98.l, v54.l, s25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v33
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v34
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v36
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s23, v129, v129
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s46, v65, v65
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v128.l, v119.l, s22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v52
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v64
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v116.l, v135.l, v68.l, s27
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v71
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v83
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v69, 16, v102
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v15
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v35.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s17, 0, v96
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s18, 0, v97
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s19, 0, v98
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v39.l, v7.h, v23.h, s29
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s41
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, s45
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v13.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v8
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v135, 16, v24
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v81, v131
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v81, 16, v53
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v80
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s20, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s57, v112, v112
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v70
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v51.l, v32.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s56, v103, v52
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v52.l, v54.l, v33.l, s22
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v99, 0xffff0000, v22
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v48.l, v6.h, v22.h, s43
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v14.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v34.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v38
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s42
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s46
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v85
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v87, v135
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s29
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v81
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s23, s4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s58, v113, v113
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s57
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v112.l, v119.l, v65.l, s23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v66
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v80, 16, v114
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s22, v85, v135
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s23, v86, v144
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s27, v101, v69
+; GFX12-TRUE16-NEXT: s_and_b32 s17, s17, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v37.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v39.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v38.l, v49.l, s7
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s29, v113, v80
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v69.l, v71.l, v51.l, s22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v71.l, v83.l, v52.l, s23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v83.l, v102.l, v64.l, s27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v13.l, v29.l, s43
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v12.l, v28.l, s45
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v9
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v96.l, v23.h, v39.l, s40
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v25
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v55.l, v71.l, v36.l, s28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v71, 16, v52
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s44, v99, v99
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v145, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v22
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s42, v86, v134
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s43
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v86, 16, v26
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s21, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s5, s24, s5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v67, 16, v12
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s58
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.h, v33.l, v35.l, s17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v49
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v38
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v85.l, v114.l, v66.l, s29
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v83
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s18, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s19, s2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v13
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v129, 16, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v53.l, v34.l, s5
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v33.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v39
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v144, 16, v96
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v84, v133
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s27
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v71
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v10.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v99.l, v22.h, v48.l, s44
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s45, v98, v145
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s47, v101, v147
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v24
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v86
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v114, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v97, v144
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v34.l, v37.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.h, v36.l, v39.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v50.l, s9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s10, 0x8000, v64.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s20, v82, v133
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v114
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v50
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v55
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v134, 16, v67
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v147, 16, v100
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v38.l, v38.l, v49.l, s20
+; GFX12-TRUE16-NEXT: s_and_b32 s10, s27, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s21, v84, v134
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s26, v99, v147
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v38
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v71
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v49.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v29
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s41
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v84, 16, v27
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s22, s3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v8.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v48
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v146, 16, v99
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v23.l, v7.l, s45
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s47
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v98
-; GFX12-TRUE16-NEXT: s_and_b32 s10, s29, s10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v67.l, v50.l, s21
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s20, 0, v99
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v10
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s46
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v52.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v65
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v130, 16, v112
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v80.l, v87.l, v53.l, s24
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s23, 0, v102
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v11.l, v27.l, s47
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s44, v128, v82
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v82.l, v100.l, v55.l, s26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v67
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s20, s3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v133, 16, v28
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.h, v52.l, v33.l, s3
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s57, v112, v114
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s10
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v11.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v66.l, v96.l, v39.l, s44
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v96, 16, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s27, 0, v84
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s46, v100, v146
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v54.l, v68.l, v35.l, s26
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v68.l, v102.l, v49.l, s56
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v100, 16, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v102, 16, v22
-; GFX12-TRUE16-NEXT: s_and_b32 s14, s43, s14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v50.l, v4.h, v20.h, s59
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s57
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v38.l, v49.l, s3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s28, v103, v130
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v80
+; GFX12-TRUE16-NEXT: s_and_b32 s6, s23, s6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v50.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, s56
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s21, 0, v100
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v24.l, v8.l, s14
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s12
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v9.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v96
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s27, s8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v7.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s18, 0x8000, v6.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s45, 0, v100
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s47, 0, v102
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.h, v71.l, v52.l, s6
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v53.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s24, 0, v103
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v30
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s21, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v29.l, v13.l, s44
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v113.l, v20.h, v50.l, s60
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v21
-; GFX12-TRUE16-NEXT: s_and_b32 s12, s41, s12
-; GFX12-TRUE16-NEXT: s_and_b32 s16, s45, s16
-; GFX12-TRUE16-NEXT: s_and_b32 s18, s47, s18
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v67.l, v50.l, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s7, s24, s7
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s43, v119, v132
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v25.l, v9.l, s12
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s57, 0, v112
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v23.l, v7.l, s16
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v22.l, v6.l, s18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v113
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v50
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v37
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v132, 16, v82
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v103, 16, v68
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v12.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s25, v67, v119
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s19, 0x8000, v49.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s40, v83, v132
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s56, 0, v103
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v83, 16, v54
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.h, v80.l, v53.l, s7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v128, 16, v29
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s16, 0x8000, v13.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v14.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v35.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v64.l, v82.l, v37.l, s40
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v65.l, v85.l, v38.l, s42
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v83
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v82, 16, v28
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v85, 16, v55
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v67.l, v99.l, v48.l, s46
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v36.l
-; GFX12-TRUE16-NEXT: s_and_b32 s7, s26, s7
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v82
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v30.l, v14.l, s43
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v68
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v128
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v131, 16, v116
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v84.l, v112.l, v65.l, s28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v119, 16, v30
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v69
+; GFX12-TRUE16-NEXT: s_and_b32 s16, s44, s16
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s41, v115, v131
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.h, v54.l, v35.l, s7
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v85
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v87, 16, v64
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v97, 16, v65
-; GFX12-TRUE16-NEXT: s_and_b32 s6, s25, s6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v99, 16, v66
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v13.l, v29.l, v13.l, s16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s43, 0, v119
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v112, 16, v81
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v87.l, v118.l, v48.l, s42
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v86.l, v116.l, v68.l, s41
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s5, 0x8000, v51.l
+; GFX12-TRUE16-NEXT: s_and_b32 s15, s43, s15
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v54.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v101, 16, v67
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s28, s9
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v37.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v38.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s15, 0x8000, v39.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s17, 0x8000, v48.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s40, 0, v87
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v97
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s44, 0, v99
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s46, 0, v101
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v14.l, v30.l, v14.l, s15
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s22, 0, v101
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s25, 0, v112
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v113, 16, v82
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v115, 16, v84
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v116, 16, v85
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v117, 16, v86
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v118, 16, v87
+; GFX12-TRUE16-NEXT: s_and_b32 s5, s22, s5
+; GFX12-TRUE16-NEXT: s_and_b32 s8, s25, s8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s9, 0x8000, v55.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s11, 0x8000, v65.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s12, 0x8000, v66.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s13, 0x8000, v68.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s14, 0x8000, v48.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s26, 0, v113
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s28, 0, v115
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s29, 0, v116
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s41, 0, v117
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s42, 0, v118
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.h, v55.l, v36.l, s9
-; GFX12-TRUE16-NEXT: s_and_b32 s11, s40, s11
-; GFX12-TRUE16-NEXT: s_and_b32 s13, s42, s13
-; GFX12-TRUE16-NEXT: s_and_b32 s15, s44, s15
-; GFX12-TRUE16-NEXT: s_and_b32 s17, s46, s17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.h, v69.l, v51.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.h, v81.l, v54.l, s8
+; GFX12-TRUE16-NEXT: s_and_b32 s9, s26, s9
+; GFX12-TRUE16-NEXT: s_and_b32 s11, s28, s11
+; GFX12-TRUE16-NEXT: s_and_b32 s12, s29, s12
+; GFX12-TRUE16-NEXT: s_and_b32 s13, s41, s13
+; GFX12-TRUE16-NEXT: s_and_b32 s14, s42, s14
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.h, v64.l, v37.l, s11
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.h, v65.l, v38.l, s13
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.h, v66.l, v39.l, s15
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.h, v67.l, v48.l, s17
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.h, v82.l, v55.l, s9
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v69, 0xffff0000, v31
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v70, 16, v31
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 16, v31
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v97, 0xffff0000, v31
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v98, 16, v31
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s40
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v15.h, v31.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v15.l, v31.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v69, v69
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v70, v70
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v33, 16, v15
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v32.l, v96.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v97, v97
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v98, v98
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v15
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v35, 16, v32
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v31.h, v32.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v96.l, v32.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v31.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v32
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.h, v84.l, v65.l, s11
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v29
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v51, 16, v30
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v31, v34
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v33, v51
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v33
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v31
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.h, v83.l, v64.l, s10
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v35, v37
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v34, v39
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v27
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v29.l, v32.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v33.l, v32.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v30.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v32.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v15.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v24
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v25
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v25, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v5.l
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v31.l, v15.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v15.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v32.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v34, 16, v33
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v35, 16, v31
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.h, v85.l, v66.l, s12
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v34
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v35
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.h, v86.l, v68.l, s13
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.h, v87.l, v48.l, s14
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s1, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v70, v70
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v26.l, v32.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v27.l, v15.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v24, v24
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s57, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v19
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.h, v33.l, v32.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v15.l, v31.l, v15.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v129, v133
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v113.l, v50.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v10.l, v26.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v3.h, v19.h, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v21.l, v5.l, s3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v28.l, v12.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v25.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v38, v38
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v37, v39
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v10
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v28
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v26
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v19.h, v22.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v50.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v3
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s56, s19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.h, v68.l, v49.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v23, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v4
-; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v26
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v28, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v27.l, v11.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v12.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v39, v38
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v27
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v11.l
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v26.l, v26.l, v10.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s3, v39, v38
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v26
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.h, v24.l, v50.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v12.l, v28.l, v12.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v18
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v20.l, v20.l, v4.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v24, 16, v21
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v19
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v9.l, s3
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s0, s2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v10.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v11.l, v27.l, v11.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v25
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v38, v38
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v39, 16, v7
+; GFX12-TRUE16-NEXT: s_and_b32 s3, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v37
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v24.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v38, v38
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v9.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v39, v39
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v23
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v10.l, v26.l, v10.l, s3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v23.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v37, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v37, 16, v24
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v38, 16, v8
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v36.l, v25.l, v9.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v2.h, v18.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v20
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v24
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v26, v25
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v22.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v23.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v26, v26
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v22
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v38, v37
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v9
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v18.h, v27.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v22.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v25, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v27
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v24.l, v8.l, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v21.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v25, v25
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
-; GFX12-TRUE16-NEXT: s_and_b32 s7, vcc_lo, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.h, v21.l, v22.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v19
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v25, v23
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v23, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v22
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v24.l, v27.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v7.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v8.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v26, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v25, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v5
+; GFX12-TRUE16-NEXT: s_and_b32 s2, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v23
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v7.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v22.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v26, v25
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v4
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v27.l, v24.l, v8.l, s2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v21.l, v5.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v28.l, v9.l, v7.l, s0
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v23, v23
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v22, 0xffff0000, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v27.l
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v20
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v21
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v24.l, v1.h, v17.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v22, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v23
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v17.h, v24.l, vcc_lo
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s3
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v3.l
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v24
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v20.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v19
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v20.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v19.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v5.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.h, v21.l, v27.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v26, v26
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v29.l, v22.l, v6.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v18.l, v18.l, v2.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v23, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s8, s1, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v23, 16, v17
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v21, 0xffff0000, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v16
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v23, v23
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v19.l, v19.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v9, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v20, v20
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s2, s4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v18
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v30.l, v21.l, v5.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v18.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v17
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v19
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v20, v20
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v23.l, v0.h, v16.h, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v21, v21
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v17.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v6, v6
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v16.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v22, v22
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, s2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v1
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v22.l, v16.h, v23.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v29, 16, v23
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v31, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v28, 16, v22
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v18
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v30, 16, v16
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v18.l, v2.l, s2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v25.l, v25.l, v24.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v27, v26
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v29, v28
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v21
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v31, v30
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v25
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v8.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v17.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v7, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v16.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v5
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v16, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v17.l, v17.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v19.l, v3.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v18, v17
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v8
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v21, v20
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v21.l, v22.l, v23.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v24.l
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v16.l, v16.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v26
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v22, 16, v17
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v26, 16, v21
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v27, 16, v16
-; GFX12-TRUE16-NEXT: s_and_b32 s9, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v22
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v26
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v23.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v27
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s6, 0x8000, v0.l
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v25.l, v24.l, s9
-; GFX12-TRUE16-NEXT: s_and_b32 s3, s5, s6
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v21.l, v23.l, s2
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v9.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v22
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v17, 16, v6
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v16
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v9
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v4.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v18, 16, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v3.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v17
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v2.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s5, 0, v16
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s6, 0, v18
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s7, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s8, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, s1
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s3, s6, s7
+; GFX12-TRUE16-NEXT: s_and_b32 s4, s5, s8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v16.l, v0.l, s3
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v17.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v18.l, v2.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v19.l, v3.l, s8
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v20.l, v4.l, s7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v35.l, v5.l, v0.l, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v34.l, v9.l, v1.l, s4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v33.l, v6.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v32.l, v7.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v31.l, v8.l, v4.l, s2
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, v35 :: v_dual_mov_b32 v1, v34
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v33 :: v_dual_mov_b32 v3, v32
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, v31 :: v_dual_mov_b32 v5, v30
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, v29 :: v_dual_mov_b32 v7, v28
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, v27 :: v_dual_mov_b32 v9, v36
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v32bf16:
@@ -12362,26 +12577,27 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -12416,17 +12632,19 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v0.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
@@ -12651,19 +12869,20 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v3.l, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
@@ -12741,21 +12960,22 @@ define <2 x bfloat> @v_minimumnum_v2bf16_no_ieee(<2 x bfloat> %x, <2 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v5, v5
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v7, v7
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.h, v1.h, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v5.l, v3.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v1.h, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s1
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, s2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
@@ -13083,58 +13303,58 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX11-TRUE16-LABEL: v_minimumnum_v3bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, v6.l, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v9
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v9, v8
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v6.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v6.l, s2
; GFX11-TRUE16-NEXT: s_and_b32 s1, s4, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee:
@@ -13202,62 +13422,65 @@ define <3 x bfloat> @v_minimumnum_v3bf16_no_ieee(<3 x bfloat> %x, <3 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v7, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v9, v9
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v0.h, v2.h, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v2.h, v4.l, s3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, v6.l, s1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v6
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v7
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v5, v9
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v7, v10
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v8, v11
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v9, v8
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v10, v11
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v6.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v6.l
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, v4.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v1.l
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v4
; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s1, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v5
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v7
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s4, 0, v8
; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, vcc_lo
; GFX12-TRUE16-NEXT: s_and_b32 s0, s3, s0
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v4.l, v6.l, s2
; GFX12-TRUE16-NEXT: s_and_b32 s1, s4, s1
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v5.l, v4.l, s0
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s2
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v3bf16_no_ieee:
@@ -13655,65 +13878,69 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v9.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v12
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s2
+; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v12
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v4.l, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX11-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.l, s5
; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v4.l, s0
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee:
@@ -13805,74 +14032,81 @@ define <4 x bfloat> @v_minimumnum_v4bf16_no_ieee(<4 x bfloat> %x, <4 x bfloat> %
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v5, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v6, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v1.h, v3.h, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s2, v8, v8
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s4, v4, v4
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s3, v9, v9
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v6.l, v5.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.h, v5.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v10, v10
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v6.l, v0.h, v2.h, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v11, v11
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v12, v12
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v9.l, v2.h, v6.l, s3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v9.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v9
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v3.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v7, v7
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s1, v8, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v5
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v9.l, v4.l, vcc_lo
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v4
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s2, v11, v12
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v13, 16, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.l, v5.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v7
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v5.l, v5.l, v6.l, s2
+; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v8, v9
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s0, v11, v10
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s2, 0x8000, v6.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s3, 0x8000, v0.l
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e64 s1, v13, v12
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, vcc_lo
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v7.l, v7.l, v4.l, vcc_lo
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v8.l, v9.l, v6.l, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v5.l
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v6.l
; GFX12-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.l, v0.l, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v8
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v3
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s4, 0x8000, v1.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v2
; GFX12-TRUE16-NEXT: s_and_b32 s5, vcc_lo, s0
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v7
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v8
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v4.l
; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s1, 0, v9
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s3, 0, v10
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s2, 0, v10
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v4.l, v5.l, s5
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.l, v6.l, s5
; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s2
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s3, s4
+; GFX12-TRUE16-NEXT: s_and_b32 s1, s1, s4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v8.l, v6.l, s1
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s2
-; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s0
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.l, v4.l, s0
+; GFX12-TRUE16-NEXT: s_and_b32 s0, s2, s3
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v3.l, v1.l, s1
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: v_minimumnum_v4bf16_no_ieee:
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
index 121175707a47f..c99f3175d5f7d 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fneg-fsub-bf16.ll
@@ -560,9 +560,8 @@ define bfloat @v_fadd_bf16_chain(bfloat %a, bfloat %b, bfloat %c) {
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v2.l
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v1
+; GFX1250-NEXT: v_pk_add_bf16 v0, v0, v2
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX1310-LABEL: v_fadd_bf16_chain:
diff --git a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
index a1e013f3400b1..a73b30e4c124d 100644
--- a/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
+++ b/llvm/test/CodeGen/AMDGPU/pseudo-scalar-transcendental.ll
@@ -51,21 +51,13 @@ define amdgpu_cs float @v_s_exp_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_exp_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: v_s_exp_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_exp_f16 s0, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-FAKE16-LABEL: v_s_exp_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_exp_f16 s0, s0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_exp_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_exp_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-TRUE16-LABEL: v_s_exp_f16:
; GFX12-GISEL-TRUE16: ; %bb.0:
@@ -103,29 +95,29 @@ define amdgpu_cs float @v_s_amdgcn_exp_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_amdgcn_exp_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: v_s_amdgcn_exp_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_exp_f16 s0, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_amdgcn_exp_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_exp_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-FAKE16-LABEL: v_s_amdgcn_exp_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_exp_f16 s0, s0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: v_s_amdgcn_exp_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_exp_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: v_s_amdgcn_exp_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_s_exp_f16 s0, s0
-; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-FAKE16-LABEL: v_s_amdgcn_exp_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_exp_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_amdgcn_exp_f16:
; GCN-GISEL: ; %bb.0:
@@ -190,21 +182,13 @@ define amdgpu_cs float @v_s_log_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_log_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: v_s_log_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_log_f16 s0, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-FAKE16-LABEL: v_s_log_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_log_f16 s0, s0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_log_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_log_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-TRUE16-LABEL: v_s_log_f16:
; GFX12-GISEL-TRUE16: ; %bb.0:
@@ -242,29 +226,29 @@ define amdgpu_cs float @v_s_amdgcn_log_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_amdgcn_log_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: v_s_amdgcn_log_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_log_f16 s0, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_amdgcn_log_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_log_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-FAKE16-LABEL: v_s_amdgcn_log_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_log_f16 s0, s0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: v_s_amdgcn_log_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_log_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: v_s_amdgcn_log_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_s_log_f16 s0, s0
-; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-FAKE16-LABEL: v_s_amdgcn_log_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_log_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_amdgcn_log_f16:
; GCN-GISEL: ; %bb.0:
@@ -373,29 +357,29 @@ define amdgpu_cs float @v_s_rcp_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_rcp_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: v_s_rcp_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_rcp_f16 s0, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_rcp_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_rcp_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-FAKE16-LABEL: v_s_rcp_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_rcp_f16 s0, s0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: v_s_rcp_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_rcp_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: v_s_rcp_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_s_rcp_f16 s0, s0
-; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-FAKE16-LABEL: v_s_rcp_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_rcp_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_rcp_f16:
; GCN-GISEL: ; %bb.0:
@@ -549,29 +533,29 @@ define amdgpu_cs float @v_s_rsq_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_rsq_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: v_s_rsq_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_rsq_f16 s0, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_rsq_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_rsq_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-FAKE16-LABEL: v_s_rsq_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_rsq_f16 s0, s0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: v_s_rsq_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_rsq_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: v_s_rsq_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_s_rsq_f16 s0, s0
-; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-FAKE16-LABEL: v_s_rsq_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_rsq_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_rsq_f16:
; GCN-GISEL: ; %bb.0:
@@ -695,29 +679,29 @@ define amdgpu_cs float @v_s_sqrt_f32(float inreg %src) {
}
define amdgpu_cs half @v_s_sqrt_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: v_s_sqrt_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_sqrt_f16 s0, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_s_sqrt_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-FAKE16-LABEL: v_s_sqrt_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_sqrt_f16 s0, s0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: v_s_sqrt_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: v_s_sqrt_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_s_sqrt_f16 s0, s0
-; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-FAKE16-LABEL: v_s_sqrt_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_s_sqrt_f16:
; GCN-GISEL: ; %bb.0:
@@ -745,29 +729,29 @@ define amdgpu_cs float @v_amdgcn_sqrt_f32(float inreg %src) {
}
define amdgpu_cs half @v_amdgcn_sqrt_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: v_amdgcn_sqrt_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_sqrt_f16 s0, s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: v_amdgcn_sqrt_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
-; GFX12-SDAG-FAKE16-LABEL: v_amdgcn_sqrt_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_sqrt_f16 s0, s0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-GISEL-TRUE16-LABEL: v_amdgcn_sqrt_f16:
+; GFX12-GISEL-TRUE16: ; %bb.0:
+; GFX12-GISEL-TRUE16-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX12-GISEL-LABEL: v_amdgcn_sqrt_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: v_s_sqrt_f16 s0, s0
-; GFX12-GISEL-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-GISEL-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
+; GFX12-GISEL-FAKE16-LABEL: v_amdgcn_sqrt_f16:
+; GFX12-GISEL-FAKE16: ; %bb.0:
+; GFX12-GISEL-FAKE16-NEXT: v_s_sqrt_f16 s0, s0
+; GFX12-GISEL-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-GISEL-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-GISEL-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GCN-GISEL-LABEL: v_amdgcn_sqrt_f16:
; GCN-GISEL: ; %bb.0:
@@ -895,21 +879,13 @@ define amdgpu_cs float @srcmods_neg_f32(float inreg %src) {
}
define amdgpu_cs half @srcmods_abs_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: srcmods_abs_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_log_f16 s0, |s0|
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-FAKE16-LABEL: srcmods_abs_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_log_f16 s0, |s0|
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: srcmods_abs_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_log_f16 s0, |s0|
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-TRUE16-LABEL: srcmods_abs_f16:
; GFX12-GISEL-TRUE16: ; %bb.0:
@@ -931,21 +907,13 @@ define amdgpu_cs half @srcmods_abs_f16(half inreg %src) {
}
define amdgpu_cs half @srcmods_neg_f16(half inreg %src) {
-; GFX12-SDAG-TRUE16-LABEL: srcmods_neg_f16:
-; GFX12-SDAG-TRUE16: ; %bb.0:
-; GFX12-SDAG-TRUE16-NEXT: v_s_log_f16 s0, -s0
-; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-FAKE16-LABEL: srcmods_neg_f16:
-; GFX12-SDAG-FAKE16: ; %bb.0:
-; GFX12-SDAG-FAKE16-NEXT: v_s_log_f16 s0, -s0
-; GFX12-SDAG-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
-; GFX12-SDAG-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-SDAG-FAKE16-NEXT: ; return to shader part epilog
+; GFX12-SDAG-LABEL: srcmods_neg_f16:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: v_s_log_f16 s0, -s0
+; GFX12-SDAG-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-TRUE16-LABEL: srcmods_neg_f16:
; GFX12-GISEL-TRUE16: ; %bb.0:
@@ -1016,7 +984,7 @@ define amdgpu_cs half @fdiv_f16_i16(half inreg %a, i16 inreg %b) {
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_2)
; GFX12-GISEL-TRUE16-NEXT: s_mul_f16 s0, s0, s1
; GFX12-GISEL-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-FAKE16-LABEL: fdiv_f16_i16:
diff --git a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
index 8fa4634d5bbe8..5f85693b2fc67 100644
--- a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
+++ b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
@@ -1366,9 +1366,9 @@ define i16 @v_mul_sub_1_i16(i16 %x, i16 %y) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -1
+; GFX1250-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -1
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_sub_1_i16:
@@ -1390,9 +1390,9 @@ define i16 @v_mul_sub_1_i16(i16 %x, i16 %y) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -1
+; GFX13-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -1
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%sub = sub i16 %y, 1
%mul = mul i16 %x, %sub
@@ -1443,9 +1443,9 @@ define i16 @v_mul_sub_1_i16_commute(i16 %x, i16 %y) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -1
+; GFX1250-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -1
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.l
+; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.h, v0.l
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_sub_1_i16_commute:
@@ -1467,9 +1467,9 @@ define i16 @v_mul_sub_1_i16_commute(i16 %x, i16 %y) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -1
+; GFX13-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -1
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.l
+; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.h, v0.l
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%sub = sub i16 %y, 1
%mul = mul i16 %sub, %x
@@ -1597,9 +1597,9 @@ define i16 @v_mul_add_2_i16(i16 %x, i16 %y) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, 2
+; GFX1250-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, 2
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_add_2_i16:
@@ -1621,9 +1621,9 @@ define i16 @v_mul_add_2_i16(i16 %x, i16 %y) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, 2
+; GFX13-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, 2
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%add = add i16 %y, 2
%mul = mul i16 %x, %add
@@ -1674,9 +1674,9 @@ define i16 @v_mul_sub_2_i16(i16 %x, i16 %y) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -2
+; GFX1250-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -2
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX1250-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_sub_2_i16:
@@ -1698,9 +1698,9 @@ define i16 @v_mul_sub_2_i16(i16 %x, i16 %y) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_add_nc_u16 v1.l, v1.l, -2
+; GFX13-REAL16-NEXT: v_add_nc_u16 v0.h, v1.l, -2
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
+; GFX13-REAL16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%sub = sub i16 %y, 2
%mul = mul i16 %x, %sub
@@ -4687,9 +4687,9 @@ define i16 @v_mul_284_add_82_i16(i16 %arg) {
; GFX1250-REAL16: ; %bb.0:
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v1.l, 0x11c
+; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, 0x11c
; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v1.l, 0x52
+; GFX1250-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, 0x52
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
;
; GFX13-FAKE16-LABEL: v_mul_284_add_82_i16:
@@ -4711,9 +4711,9 @@ define i16 @v_mul_284_add_82_i16(i16 %arg) {
; GFX13-REAL16-NEXT: s_wait_samplecnt 0x0
; GFX13-REAL16-NEXT: s_wait_bvhcnt 0x0
; GFX13-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX13-REAL16-NEXT: v_mov_b16_e32 v1.l, 0x11c
+; GFX13-REAL16-NEXT: v_mov_b16_e32 v0.h, 0x11c
; GFX13-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX13-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v1.l, 0x52
+; GFX13-REAL16-NEXT: v_mad_u16 v0.l, v0.l, v0.h, 0x52
; GFX13-REAL16-NEXT: s_set_pc_i64 s[30:31]
%mul = mul i16 %arg, 284
%add = add i16 %mul, 82
diff --git a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
index f98d30449fe3d..b53b0736ea615 100644
--- a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
+++ b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
@@ -282,10 +282,10 @@ define <2 x half> @v_repeat_divisor_f16_x2_arcp(half %x, half %y, half %D) #0 {
; GFX11-TRUE16-LABEL: v_repeat_divisor_f16_x2_arcp:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v2.l, v2.l
+; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.h, v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_repeat_divisor_f16_x2_arcp:
@@ -569,11 +569,11 @@ define <3 x half> @v_repeat_divisor_f16_x3_arcp(half %x, half %y, half %z, half
; GFX11-TRUE16-LABEL: v_repeat_divisor_f16_x3_arcp:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v3.l, v3.l
+; GFX11-TRUE16-NEXT: v_rcp_f16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.h, v1.l, v3.l
-; GFX11-TRUE16-NEXT: v_mul_f16_e32 v1.l, v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_mul_f16_e32 v1.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_repeat_divisor_f16_x3_arcp:
diff --git a/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll b/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
index 7ce198d633e02..119024a395946 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -mtriple=amdgpu11.50 < %s | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-GFX11
+; RUN: llc -global-isel=0 -mtriple=amdgpu11.50 < %s | FileCheck %s --check-prefixes=CHECK,SDAG
; RUN: llc -global-isel=1 -mtriple=amdgpu11.50 < %s | FileCheck %s --check-prefixes=CHECK,GISEL-GFX11
-; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 < %s | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-GFX12
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 < %s | FileCheck %s --check-prefixes=CHECK,SDAG
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck %s --check-prefixes=CHECK,GISEL-GFX12
define amdgpu_vs float @sitofp_i32_to_f32(i32 inreg %val) {
@@ -106,21 +106,21 @@ define amdgpu_vs half @fptrunc_f32_to_f16(float inreg %val) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_cvt_f16_f32 s0, s0
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
; SDAG-NEXT: ; return to shader part epilog
;
; GISEL-GFX11-LABEL: fptrunc_f32_to_f16:
; GISEL-GFX11: ; %bb.0:
; GISEL-GFX11-NEXT: s_cvt_f16_f32 s0, s0
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX11-NEXT: ; return to shader part epilog
;
; GISEL-GFX12-LABEL: fptrunc_f32_to_f16:
; GISEL-GFX12: ; %bb.0:
; GISEL-GFX12-NEXT: s_cvt_f16_f32 s0, s0
; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GISEL-GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = fptrunc float %val to half
ret half %res
@@ -185,21 +185,21 @@ define amdgpu_vs half @fceil_f16(half inreg %val) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_ceil_f16 s0, s0
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
; SDAG-NEXT: ; return to shader part epilog
;
; GISEL-GFX11-LABEL: fceil_f16:
; GISEL-GFX11: ; %bb.0:
; GISEL-GFX11-NEXT: s_ceil_f16 s0, s0
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX11-NEXT: ; return to shader part epilog
;
; GISEL-GFX12-LABEL: fceil_f16:
; GISEL-GFX12: ; %bb.0:
; GISEL-GFX12-NEXT: s_ceil_f16 s0, s0
; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GISEL-GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.ceil.f16(half %val)
ret half %res
@@ -210,21 +210,21 @@ define amdgpu_vs half @ffloor_f16(half inreg %val) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_floor_f16 s0, s0
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
; SDAG-NEXT: ; return to shader part epilog
;
; GISEL-GFX11-LABEL: ffloor_f16:
; GISEL-GFX11: ; %bb.0:
; GISEL-GFX11-NEXT: s_floor_f16 s0, s0
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX11-NEXT: ; return to shader part epilog
;
; GISEL-GFX12-LABEL: ffloor_f16:
; GISEL-GFX12: ; %bb.0:
; GISEL-GFX12-NEXT: s_floor_f16 s0, s0
; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GISEL-GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.floor.f16(half %val)
ret half %res
@@ -235,21 +235,21 @@ define amdgpu_vs half @ftrunc_f16(half inreg %val) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_trunc_f16 s0, s0
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
; SDAG-NEXT: ; return to shader part epilog
;
; GISEL-GFX11-LABEL: ftrunc_f16:
; GISEL-GFX11: ; %bb.0:
; GISEL-GFX11-NEXT: s_trunc_f16 s0, s0
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX11-NEXT: ; return to shader part epilog
;
; GISEL-GFX12-LABEL: ftrunc_f16:
; GISEL-GFX12: ; %bb.0:
; GISEL-GFX12-NEXT: s_trunc_f16 s0, s0
; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GISEL-GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = call half @llvm.trunc.f16(half %val)
ret half %res
@@ -260,7 +260,7 @@ define amdgpu_vs half @frint_f16(half inreg %val) {
; SDAG: ; %bb.0:
; SDAG-NEXT: s_rndne_f16 s0, s0
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
; SDAG-NEXT: ; return to shader part epilog
;
; GISEL-GFX11-LABEL: frint_f16:
@@ -304,7 +304,7 @@ define amdgpu_vs half @sitofp_i32_to_f16(i32 inreg %x) {
; SDAG-NEXT: s_cvt_f32_i32 s0, s0
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
; SDAG-NEXT: s_cvt_f16_f32 s0, s0
-; SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
; SDAG-NEXT: ; return to shader part epilog
;
; GISEL-GFX11-LABEL: sitofp_i32_to_f16:
@@ -312,7 +312,7 @@ define amdgpu_vs half @sitofp_i32_to_f16(i32 inreg %x) {
; GISEL-GFX11-NEXT: s_cvt_f32_i32 s0, s0
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
; GISEL-GFX11-NEXT: s_cvt_f16_f32 s0, s0
-; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX11-NEXT: ; return to shader part epilog
;
; GISEL-GFX12-LABEL: sitofp_i32_to_f16:
@@ -320,7 +320,7 @@ define amdgpu_vs half @sitofp_i32_to_f16(i32 inreg %x) {
; GISEL-GFX12-NEXT: s_cvt_f32_i32 s0, s0
; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
; GISEL-GFX12-NEXT: s_cvt_f16_f32 s0, s0
-; GISEL-GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = sitofp i32 %x to half
ret half %res
@@ -332,7 +332,7 @@ define amdgpu_vs half @uitofp_i32_to_f16(i32 inreg %x) {
; SDAG-NEXT: s_cvt_f32_u32 s0, s0
; SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
; SDAG-NEXT: s_cvt_f16_f32 s0, s0
-; SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; SDAG-NEXT: v_mov_b32_e32 v0, s0
; SDAG-NEXT: ; return to shader part epilog
;
; GISEL-GFX11-LABEL: uitofp_i32_to_f16:
@@ -340,7 +340,7 @@ define amdgpu_vs half @uitofp_i32_to_f16(i32 inreg %x) {
; GISEL-GFX11-NEXT: s_cvt_f32_u32 s0, s0
; GISEL-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
; GISEL-GFX11-NEXT: s_cvt_f16_f32 s0, s0
-; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX11-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX11-NEXT: ; return to shader part epilog
;
; GISEL-GFX12-LABEL: uitofp_i32_to_f16:
@@ -348,7 +348,7 @@ define amdgpu_vs half @uitofp_i32_to_f16(i32 inreg %x) {
; GISEL-GFX12-NEXT: s_cvt_f32_u32 s0, s0
; GISEL-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
; GISEL-GFX12-NEXT: s_cvt_f16_f32 s0, s0
-; GISEL-GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GISEL-GFX12-NEXT: v_mov_b16_e32 v0.l, s0
; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = uitofp i32 %x to half
ret half %res
diff --git a/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll b/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
index 43a1bcf934506..c2103705f0dd8 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar-float-sop2.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgpu11.50 < %s | FileCheck -check-prefixes=CHECK,GFX1150,GFX1150-SDAG %s
-; RUN: llc -mtriple=amdgpu11.50 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX1150,GFX1150-GISEL %s
-; RUN: llc -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=CHECK,GFX12,GFX12-SDAG %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX12,GFX12-GISEL %s
+; RUN: llc -mtriple=amdgpu11.50 < %s | FileCheck -check-prefixes=CHECK,GFX1150 %s
+; RUN: llc -mtriple=amdgpu11.50 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX1150 %s
+; RUN: llc -mtriple=amdgpu12.00 < %s | FileCheck -check-prefixes=CHECK,GFX12 %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel < %s | FileCheck -check-prefixes=CHECK,GFX12 %s
define amdgpu_vs float @fadd_f32(float inreg %a, float inreg %b) {
; CHECK-LABEL: fadd_f32:
@@ -74,161 +74,26 @@ define amdgpu_vs float @fmax_f32(float inreg %a, float inreg %b) {
}
define amdgpu_vs half @fadd_f16(half inreg %a, half inreg %b) {
-; GFX1150-SDAG-LABEL: fadd_f16:
-; GFX1150-SDAG: ; %bb.0:
-; GFX1150-SDAG-NEXT: s_add_f16 s0, s0, s1
-; GFX1150-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1150-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX1150-GISEL-LABEL: fadd_f16:
-; GFX1150-GISEL: ; %bb.0:
-; GFX1150-GISEL-NEXT: s_add_f16 s0, s0, s1
-; GFX1150-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1150-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-LABEL: fadd_f16:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: s_add_f16 s0, s0, s1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: fadd_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_add_f16 s0, s0, s1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
%add = fadd half %a, %b
ret half %add
}
define amdgpu_vs half @fsub_f16(half inreg %a, half inreg %b) {
-; GFX1150-SDAG-LABEL: fsub_f16:
-; GFX1150-SDAG: ; %bb.0:
-; GFX1150-SDAG-NEXT: s_sub_f16 s0, s0, s1
-; GFX1150-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1150-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX1150-GISEL-LABEL: fsub_f16:
-; GFX1150-GISEL: ; %bb.0:
-; GFX1150-GISEL-NEXT: s_sub_f16 s0, s0, s1
-; GFX1150-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1150-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-LABEL: fsub_f16:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: s_sub_f16 s0, s0, s1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: fsub_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_sub_f16 s0, s0, s1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
%sub = fsub half %a, %b
ret half %sub
}
define amdgpu_vs half @fmul_f16(half inreg %a, half inreg %b) {
-; GFX1150-SDAG-LABEL: fmul_f16:
-; GFX1150-SDAG: ; %bb.0:
-; GFX1150-SDAG-NEXT: s_mul_f16 s0, s0, s1
-; GFX1150-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1150-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX1150-GISEL-LABEL: fmul_f16:
-; GFX1150-GISEL: ; %bb.0:
-; GFX1150-GISEL-NEXT: s_mul_f16 s0, s0, s1
-; GFX1150-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1150-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-LABEL: fmul_f16:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: s_mul_f16 s0, s0, s1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: fmul_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_mul_f16 s0, s0, s1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
%mul = fmul half %a, %b
ret half %mul
}
define amdgpu_vs half @fmin_f16(half inreg %a, half inreg %b) {
-; GFX1150-SDAG-LABEL: fmin_f16:
-; GFX1150-SDAG: ; %bb.0:
-; GFX1150-SDAG-NEXT: s_min_f16 s0, s0, s1
-; GFX1150-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1150-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX1150-GISEL-LABEL: fmin_f16:
-; GFX1150-GISEL: ; %bb.0:
-; GFX1150-GISEL-NEXT: s_min_f16 s0, s0, s1
-; GFX1150-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1150-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-LABEL: fmin_f16:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: s_min_num_f16 s0, s0, s1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: fmin_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_min_num_f16 s0, s0, s1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
%min = call half @llvm.minnum.f16(half %a, half %b)
ret half %min
}
define amdgpu_vs half @fmax_f16(half inreg %a, half inreg %b) {
-; GFX1150-SDAG-LABEL: fmax_f16:
-; GFX1150-SDAG: ; %bb.0:
-; GFX1150-SDAG-NEXT: s_max_f16 s0, s0, s1
-; GFX1150-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1150-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX1150-GISEL-LABEL: fmax_f16:
-; GFX1150-GISEL: ; %bb.0:
-; GFX1150-GISEL-NEXT: s_max_f16 s0, s0, s1
-; GFX1150-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1150-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-LABEL: fmax_f16:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: s_max_num_f16 s0, s0, s1
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: fmax_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_max_num_f16 s0, s0, s1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
%max = call half @llvm.maxnum.f16(half %a, half %b)
ret half %max
}
@@ -273,78 +138,12 @@ define amdgpu_vs float @fmac_f32_with_mov(float inreg %a, float inreg %b, float
}
define amdgpu_vs half @fmac_f16(half inreg %a, half inreg %b, half inreg %c) {
-; GFX1150-SDAG-LABEL: fmac_f16:
-; GFX1150-SDAG: ; %bb.0:
-; GFX1150-SDAG-NEXT: s_fmac_f16 s0, s1, s2
-; GFX1150-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1150-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX1150-GISEL-LABEL: fmac_f16:
-; GFX1150-GISEL: ; %bb.0:
-; GFX1150-GISEL-NEXT: s_fmac_f16 s0, s1, s2
-; GFX1150-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1150-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-LABEL: fmac_f16:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: s_fmac_f16 s0, s1, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: fmac_f16:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_fmac_f16 s0, s1, s2
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
%res = call half @llvm.fma.f16(half %b, half %c, half %a)
ret half %res
}
; Check selection of mov + fmac if src2 of fmac has a use later
define amdgpu_vs half @fmac_f16_with_mov(half inreg %a, half inreg %b, half inreg %c) {
-; GFX1150-SDAG-LABEL: fmac_f16_with_mov:
-; GFX1150-SDAG: ; %bb.0:
-; GFX1150-SDAG-NEXT: s_mov_b32 s3, s2
-; GFX1150-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_3)
-; GFX1150-SDAG-NEXT: s_fmac_f16 s3, s0, s1
-; GFX1150-SDAG-NEXT: s_add_f16 s0, s3, s2
-; GFX1150-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1150-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX1150-GISEL-LABEL: fmac_f16_with_mov:
-; GFX1150-GISEL: ; %bb.0:
-; GFX1150-GISEL-NEXT: s_mov_b32 s3, s2
-; GFX1150-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_3)
-; GFX1150-GISEL-NEXT: s_fmac_f16 s3, s0, s1
-; GFX1150-GISEL-NEXT: s_add_f16 s0, s3, s2
-; GFX1150-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1150-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX1150-GISEL-NEXT: ; return to shader part epilog
-;
-; GFX12-SDAG-LABEL: fmac_f16_with_mov:
-; GFX12-SDAG: ; %bb.0:
-; GFX12-SDAG-NEXT: s_mov_b32 s3, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: s_fmac_f16 s3, s0, s1
-; GFX12-SDAG-NEXT: s_add_f16 s0, s3, s2
-; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-SDAG-NEXT: ; return to shader part epilog
-;
-; GFX12-GISEL-LABEL: fmac_f16_with_mov:
-; GFX12-GISEL: ; %bb.0:
-; GFX12-GISEL-NEXT: s_mov_b32 s3, s2
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: s_fmac_f16 s3, s0, s1
-; GFX12-GISEL-NEXT: s_add_f16 s0, s3, s2
-; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GISEL-NEXT: ; return to shader part epilog
%fma = call half @llvm.fma.f16(half %a, half %b, half %c)
%res = fadd half %fma, %c
ret half %res
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index da789a9712d50..aa4952bf97674 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -463,16 +463,12 @@ define i64 @bitcast_combine_scalar_to_vector_v4i16(i16 %arg) {
; GFX11-LABEL: bitcast_combine_scalar_to_vector_v4i16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b16 v1.h, 0xff00, v1.l
-; GFX11-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
-; GFX11-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-NEXT: v_mov_b16_e32 v1.h, v2.h
-; GFX11-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-NEXT: v_and_b16 v0.h, 0xff00, v0.l
+; GFX11-NEXT: v_lshrrev_b16 v1.l, 8, v0.l
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_or_b16 v0.h, v1.l, v0.h
+; GFX11-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-NEXT: v_mov_b16_e32 v1.h, v0.h
; GFX11-NEXT: s_setpc_b64 s[30:31]
%arg.cast = bitcast i16 %arg to <2 x i8>
%tmp1 = shufflevector <2 x i8> %arg.cast, <2 x i8> poison, <8 x i32> <i32 0, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
diff --git a/llvm/test/CodeGen/AMDGPU/scmp.ll b/llvm/test/CodeGen/AMDGPU/scmp.ll
index 6d567d5790caf..639c6368022ca 100644
--- a/llvm/test/CodeGen/AMDGPU/scmp.ll
+++ b/llvm/test/CodeGen/AMDGPU/scmp.ll
@@ -505,29 +505,29 @@ define <4 x i32> @scmp_v4i8(<4 x i8> %a, <4 x i8> %b) {
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
; GFX12-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-NEXT: v_bfe_i32 v8, v3, 0, 8
+; GFX12-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
; GFX12-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX12-SDAG-NEXT: v_bfe_i32 v8, v3, 0, 8
+; GFX12-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
; GFX12-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.l, v4.l
-; GFX12-SDAG-NEXT: v_bfe_i32 v4, v6, 0, 8
+; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v0.l, v4.l
; GFX12-SDAG-NEXT: v_bfe_i32 v9, v2, 0, 8
; GFX12-SDAG-NEXT: v_mov_b16_e32 v2.l, v5.l
; GFX12-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v0.l, v3.l
; GFX12-SDAG-NEXT: v_mov_b16_e32 v5.l, v8.l
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc_lo
-; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v0.l, v3.l
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc_lo
+; GFX12-SDAG-NEXT: v_cmp_ge_i16_e32 vcc_lo, v0.l, v4.l
+; GFX12-SDAG-NEXT: v_mov_b16_e32 v4.l, v6.l
+; GFX12-SDAG-NEXT: v_mov_b16_e32 v6.l, v7.l
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v6, vcc_lo
+; GFX12-SDAG-NEXT: v_cndmask_b32_e32 v0, -1, v3, vcc_lo
+; GFX12-SDAG-NEXT: v_mov_b16_e32 v3.l, v9.l
; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v1.l, v2.l
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v6.l, v7.l
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc_lo
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-SDAG-NEXT: v_cmp_gt_i16_e32 vcc_lo, v3.l, v4.l
; GFX12-SDAG-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.f16.ll b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.f16.ll
index ef5767738c328..1b71e4a27637d 100644
--- a/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-fabs-fneg-extract.f16.ll
@@ -337,10 +337,10 @@ define half @add_select_fabs_negk_negk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_fabs_negk_negk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xbc00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0xc000, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xbc00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xc000, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_sub_f16_e32 v0.l, v1.l, v0.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -387,10 +387,10 @@ define half @add_select_posk_posk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_posk_posk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0x3c00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0x4000, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x3c00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0x4000, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -436,10 +436,10 @@ define half @add_select_negk_fabs_f16(i32 %c, half %x, half %y) {
; GFX11-SAFE-TRUE16-LABEL: add_select_negk_fabs_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SAFE-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v1.l
; GFX11-SAFE-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v1.l
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xbc00, v0.l, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xbc00, v1.l, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -486,10 +486,10 @@ define half @add_select_negliteralk_fabs_f16(i32 %c, half %x, half %y) {
; GFX11-SAFE-TRUE16-LABEL: add_select_negliteralk_fabs_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SAFE-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v1.l
; GFX11-SAFE-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v1.l
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xe400, v0.l, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xe400, v1.l, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1021,10 +1021,10 @@ define half @add_select_negk_negk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_negk_negk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xbc00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0xc000, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xbc00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xc000, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1070,10 +1070,10 @@ define half @add_select_negliteralk_negliteralk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_negliteralk_negliteralk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xec00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0xe800, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xec00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xe800, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1119,10 +1119,10 @@ define half @add_select_fneg_negk_negk_f16(i32 %c, half %x) {
; GFX11-SAFE-TRUE16-LABEL: add_select_fneg_negk_negk_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xbc00
; GFX11-SAFE-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v2.l, 0xc000, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0xbc00
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, 0xc000, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_sub_f16_e32 v0.l, v1.l, v0.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1672,10 +1672,10 @@ define half @mul_select_posk_negfabs_f16(i32 %c, half %x, half %y) {
; GFX11-SAFE-TRUE16-LABEL: mul_select_posk_negfabs_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SAFE-TRUE16-NEXT: v_or_b16 v1.l, 0x8000, v1.l
; GFX11-SAFE-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: v_or_b16 v0.l, 0x8000, v1.l
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4400, v0.l, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4400, v1.l, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1774,10 +1774,10 @@ define half @mul_select_negk_negfabs_f16(i32 %c, half %x, half %y) {
; GFX11-SAFE-TRUE16-LABEL: mul_select_negk_negfabs_f16:
; GFX11-SAFE-TRUE16: ; %bb.0:
; GFX11-SAFE-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SAFE-TRUE16-NEXT: v_or_b16 v1.l, 0x8000, v1.l
; GFX11-SAFE-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
-; GFX11-SAFE-TRUE16-NEXT: v_or_b16 v0.l, 0x8000, v1.l
-; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xc400, v0.l, vcc_lo
+; GFX11-SAFE-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SAFE-TRUE16-NEXT: v_cndmask_b16 v0.l, 0xc400, v1.l, vcc_lo
; GFX11-SAFE-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-SAFE-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll b/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
index 8e7d95e7ce2f5..20e8d8e522d8f 100644
--- a/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
+++ b/llvm/test/CodeGen/AMDGPU/srl-bitcast-bv.ll
@@ -65,8 +65,8 @@ define i64 @srl_bv_not_last(i16 %a, i16 %b, i16 %c, i16 %d) {
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX1250-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX1250-NEXT: v_cvt_u32_u16_e32 v1, v3.l
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_cvt_u32_u16_e32 v1, v2.h
; GFX1250-NEXT: v_alignbit_b32 v0, v2, v0, 16
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%bv0 = insertelement <4 x i16> poison, i16 %a, i32 0
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
index 86c51285eb18c..0d2a8bd135101 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
@@ -524,14 +524,14 @@ define amdgpu_ps half @s_constained_fadd_f16_fpexcept_strict(half inreg %x, half
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_add_f16 s0, s2, s3
; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: s_constained_fadd_f16_fpexcept_strict:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_add_f16 s0, s2, s3
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call half @llvm.experimental.constrained.fadd.f16(half %x, half %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret half %val
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
index 3a4b9ec9a187f..821c48948beda 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
@@ -596,14 +596,14 @@ define amdgpu_ps half @s_constained_fmul_f16_fpexcept_strict(half inreg %x, half
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_mul_f16 s0, s2, s3
; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: s_constained_fmul_f16_fpexcept_strict:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_mul_f16 s0, s2, s3
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-GISEL-NEXT: ; return to shader part epilog
; GFX1-GISEL-TRUE16-LABEL: s_constained_fmul_f16_fpexcept_strict:
; GFX1-GISEL-TRUE16: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 7f853e1c109ed..1a16f39de9815 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -736,14 +736,14 @@ define amdgpu_ps half @s_constained_fsub_f16_fpexcept_strict(half inreg %x, half
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_sub_f16 s0, s2, s3
; GFX12-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-SDAG-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, s0
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: s_constained_fsub_f16_fpexcept_strict:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_sub_f16 s0, s2, s3
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-GISEL-NEXT: v_mov_b16_e32 v0.l, s0
; GFX12-GISEL-NEXT: ; return to shader part epilog
%val = call half @llvm.experimental.constrained.fsub.f16(half %x, half %y, metadata !"round.tonearest", metadata !"fpexcept.strict")
ret half %val
diff --git a/llvm/test/CodeGen/AMDGPU/usubsat.ll b/llvm/test/CodeGen/AMDGPU/usubsat.ll
index eddce27deac44..fc8b4a1b78694 100644
--- a/llvm/test/CodeGen/AMDGPU/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/usubsat.ll
@@ -50,10 +50,11 @@ define i8 @s_usubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
; GFX11-TRUE16-LABEL: s_usubsat_i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b32_e64 v0, 0xff, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e64 v1, 0xff, s0
-; GFX11-TRUE16-NEXT: v_max_u32_e32 v1, v1, v0
-; GFX11-TRUE16-NEXT: v_sub_nc_u32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: s_usubsat_i8:
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
index 297d43357c376..c3479d87e5c62 100644
--- a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
@@ -8,22 +8,20 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX11-TRUE16-LABEL: swap:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_swap_b16 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_swap_b16 v1.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %ret
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: swap:
@@ -52,15 +50,13 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX12-TRUE16-NEXT: v_swap_b16 v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_swap_b16 v1.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
@@ -69,6 +65,7 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %ret
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: swap:
@@ -116,32 +113,30 @@ define half @swap_B(half %a, half %b, half %c, i32 %i) {
; GFX11-TRUE16-LABEL: swap_B:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB1_1: ; %loop
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, -1, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: ;;#ASMSTART
; GFX11-TRUE16-NEXT: ; use v0.l
; GFX11-TRUE16-NEXT: ;;#ASMEND
-; GFX11-TRUE16-NEXT: v_swap_b16 v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_swap_b16 v1.l, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
; GFX11-TRUE16-NEXT: ;;#ASMSTART
-; GFX11-TRUE16-NEXT: ; use v1.l
+; GFX11-TRUE16-NEXT: ; use v0.h
; GFX11-TRUE16-NEXT: ;;#ASMEND
; GFX11-TRUE16-NEXT: ;;#ASMSTART
; GFX11-TRUE16-NEXT: ; use v2.l
; GFX11-TRUE16-NEXT: ;;#ASMEND
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %ret
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: swap_B:
@@ -179,25 +174,24 @@ define half @swap_B(half %a, half %b, half %c, i32 %i) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB1_1: ; %loop
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v3, -1, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX12-TRUE16-NEXT: ;;#ASMSTART
; GFX12-TRUE16-NEXT: ; use v0.l
; GFX12-TRUE16-NEXT: ;;#ASMEND
-; GFX12-TRUE16-NEXT: v_swap_b16 v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_swap_b16 v1.l, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
; GFX12-TRUE16-NEXT: ;;#ASMSTART
-; GFX12-TRUE16-NEXT: ; use v1.l
+; GFX12-TRUE16-NEXT: ; use v0.h
; GFX12-TRUE16-NEXT: ;;#ASMEND
; GFX12-TRUE16-NEXT: ;;#ASMSTART
; GFX12-TRUE16-NEXT: ; use v2.l
; GFX12-TRUE16-NEXT: ;;#ASMEND
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -205,7 +199,7 @@ define half @swap_B(half %a, half %b, half %c, i32 %i) {
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB1_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %ret
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: swap_B:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
index 5da1363b30397..c015df52f85f3 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
@@ -915,22 +915,22 @@ define i8 @test_vector_reduce_add_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_add_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v2.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
@@ -1029,22 +1029,22 @@ define i8 @test_vector_reduce_add_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.l, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v3.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v2.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v1.l, v2.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
index 7bd805dd8c575..0ade3948eb61d 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
@@ -813,23 +813,23 @@ define i8 @test_vector_reduce_and_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_and_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.h, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v3.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -890,23 +890,23 @@ define i8 @test_vector_reduce_and_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.h, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, v2.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, v3.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
index 71baa8abd2886..358d4dcd8739e 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
@@ -70,8 +70,9 @@ define half @test_vector_reduce_fadd_v2half(half %sp, <2 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fadd_v2half:
@@ -108,8 +109,9 @@ define half @test_vector_reduce_fadd_v2half(half %sp, <2 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_fadd_v2half:
@@ -219,8 +221,9 @@ define half @test_vector_reduce_fadd_v3half(half %sp, <3 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -261,8 +264,9 @@ define half @test_vector_reduce_fadd_v3half(half %sp, <3 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -389,11 +393,13 @@ define half @test_vector_reduce_fadd_v4half(half %sp, <4 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fadd_v4half:
@@ -441,11 +447,13 @@ define half @test_vector_reduce_fadd_v4half(half %sp, <4 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_fadd_v4half:
@@ -627,17 +635,21 @@ define half @test_vector_reduce_fadd_v8half(half %sp, <8 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fadd_v8half:
@@ -707,17 +719,21 @@ define half @test_vector_reduce_fadd_v8half(half %sp, <8 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_fadd_v8half:
@@ -1017,29 +1033,37 @@ define half @test_vector_reduce_fadd_v16half(half %sp, <16 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v6.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v7.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v8.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v8.h
+; GFX11-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fadd_v16half:
@@ -1153,29 +1177,37 @@ define half @test_vector_reduce_fadd_v16half(half %sp, <16 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v6.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v7.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v8.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v8.h
+; GFX12-SDAG-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_fadd_v16half:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
index dc44dec957332..b305943b6ed1d 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
@@ -70,8 +70,9 @@ define half @test_vector_reduce_fmul_v2half(half %sp, <2 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fmul_v2half:
@@ -108,8 +109,9 @@ define half @test_vector_reduce_fmul_v2half(half %sp, <2 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_fmul_v2half:
@@ -219,8 +221,9 @@ define half @test_vector_reduce_fmul_v3half(half %sp, <3 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -261,8 +264,9 @@ define half @test_vector_reduce_fmul_v3half(half %sp, <3 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -389,11 +393,13 @@ define half @test_vector_reduce_fmul_v4half(half %sp, <4 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fmul_v4half:
@@ -441,11 +447,13 @@ define half @test_vector_reduce_fmul_v4half(half %sp, <4 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_fmul_v4half:
@@ -627,17 +635,21 @@ define half @test_vector_reduce_fmul_v8half(half %sp, <8 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fmul_v8half:
@@ -707,17 +719,21 @@ define half @test_vector_reduce_fmul_v8half(half %sp, <8 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_fmul_v8half:
@@ -1017,29 +1033,37 @@ define half @test_vector_reduce_fmul_v16half(half %sp, <16 x half> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v6.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v7.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v8.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v8.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_fmul_v16half:
@@ -1153,29 +1177,37 @@ define half @test_vector_reduce_fmul_v16half(half %sp, <16 x half> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v6.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v7.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v8.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v8.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_fmul_v16half:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
index d64d59f39f273..663919584701e 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
@@ -902,23 +902,23 @@ define i8 @test_vector_reduce_mul_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.h, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.h, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1004,23 +1004,23 @@ define i8 @test_vector_reduce_mul_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.h, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.h, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
index d7db9c845b4ad..9e3e273d4253b 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
@@ -841,23 +841,23 @@ define i8 @test_vector_reduce_or_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v3.h, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -915,23 +915,23 @@ define i8 @test_vector_reduce_or_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v3.l, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v3.h, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.h, v3.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v1.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
index 659882d67d6c8..76d9d529ee725 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
@@ -208,16 +208,11 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v0.l, 0xff80
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -249,16 +244,11 @@ define i8 @test_vector_reduce_smax_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v0.l, 0xff80
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -442,11 +432,11 @@ define i8 @test_vector_reduce_smax_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -510,11 +500,11 @@ define i8 @test_vector_reduce_smax_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -776,31 +766,27 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v5.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v5
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -879,31 +865,27 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v1.l, v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v5.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v5
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1254,50 +1236,42 @@ define i8 @test_vector_reduce_smax_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v5, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v9.l, v14.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v10, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.h, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v18, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v7.l, v8.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v4.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v4.l
; GFX11-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v5.l, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -1405,50 +1379,42 @@ define i8 @test_vector_reduce_smax_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v5, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v9.l, v14.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v10, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.l, v1.h, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v18, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.h, v7.l, v8.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v4.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.h, v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_max3_i16 v0.l, v0.h, v1.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.h, v2.l, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v7
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v1.l, v1.l, v4.l
; GFX12-SDAG-TRUE16-NEXT: v_max_i16 v0.l, v5.l, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
index 09f08d332793e..1fa58fe40b246 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
@@ -208,16 +208,11 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v0.l, 0x7f
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -249,16 +244,11 @@ define i8 @test_vector_reduce_smin_v3i8(<3 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v0.l, 0x7f
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -442,11 +432,11 @@ define i8 @test_vector_reduce_smin_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -510,11 +500,11 @@ define i8 @test_vector_reduce_smin_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v1, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
@@ -776,31 +766,27 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v5.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v5.l, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v5
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -879,31 +865,27 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v1.l, v5.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v5.l, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v5
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1254,50 +1236,42 @@ define i8 @test_vector_reduce_smin_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v4, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v5, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v0, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v4.l, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v9.l, v14.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v10, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.h, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v18, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v7.l, v8.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v4.l, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v1.l, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v7
+; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v4.l
; GFX11-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v5.l, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
@@ -1405,50 +1379,42 @@ define i8 @test_vector_reduce_smin_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v16, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v4, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v5, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v18.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v0, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v17, v5, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v9, v8, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v11, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v7, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v15, 0, 8
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v3, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v6.l, v13.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v4.l, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v17.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v9.l, v14.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v6, v10, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.l, v1.h, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v18, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v9, 0, 8
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.h, v7.l, v8.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v11, v0, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v4.l, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.h, v0.h, v1.l, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v5, v2, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_min3_i16 v0.l, v0.h, v1.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX12-SDAG-TRUE16-NEXT: v_bfe_i32 v4, v14, 0, 8
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.h, v2.l, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v7
+; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v1.l, v1.l, v4.l
; GFX12-SDAG-TRUE16-NEXT: v_min_i16 v0.l, v5.l, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
index 2a587008515e5..1c3b8d88adc1d 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
@@ -412,15 +412,15 @@ define i8 @test_vector_reduce_umax_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v4i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v0.h, v1.l
@@ -477,15 +477,15 @@ define i8 @test_vector_reduce_umax_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v0.h, v1.l
@@ -727,23 +727,23 @@ define i8 @test_vector_reduce_umax_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v2.h, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
@@ -816,23 +816,23 @@ define i8 @test_vector_reduce_umax_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v1.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v2.h, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
@@ -1181,34 +1181,34 @@ define i8 @test_vector_reduce_umax_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v13.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v8.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v8.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v0.h, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v12.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v10.l
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v10.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v3.h, v6.h, v3.h
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v2.h, v3.l, v2.h
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v5.l, v5.h
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v6.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v1.h, v2.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v6.l, v7.l, v6.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v3.l, v3.l, v5.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v4.h
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v1.l, v1.l, v3.h, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v2.h, v4.l, v7.h
+; GFX11-SDAG-TRUE16-NEXT: v_max_u16 v2.l, v2.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v4.l, v4.h
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v2.h, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v6.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v2.h
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
@@ -1314,34 +1314,34 @@ define i8 @test_vector_reduce_umax_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v13.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v8.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v9.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v8.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.h, v0.h, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v12.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v10.l
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v1.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v10.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v3.h, v6.h, v3.h
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v2.h, v3.l, v2.h
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v5.l, v5.h
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.l, v6.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v1.h, v2.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v6.l, v7.l, v6.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v3.l, v3.l, v5.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v0.l, v0.l, v4.h
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v1.l, v1.l, v3.h, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v2.h, v4.l, v7.h
+; GFX12-SDAG-TRUE16-NEXT: v_max_u16 v2.l, v2.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v4.l, v4.h
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v0.h, v2.h, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.h, v1.l, v3.l, v6.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_max3_u16 v0.l, v0.l, v2.l, v2.h
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
index 77ea7b6d8df87..b893df9081b75 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
@@ -367,15 +367,15 @@ define i8 @test_vector_reduce_umin_v4i8(<4 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v4i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v0.h, v1.l
@@ -419,15 +419,15 @@ define i8 @test_vector_reduce_umin_v4i8(<4 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v1.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v1, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v0.h, v1.l
@@ -613,23 +613,23 @@ define i8 @test_vector_reduce_umin_v8i8(<8 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v2.h, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
@@ -688,23 +688,23 @@ define i8 @test_vector_reduce_umin_v8i8(<8 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v1.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v2.h, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v6.l
; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_cvt_u32_u16_e32 v3, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
@@ -996,34 +996,34 @@ define i8 @test_vector_reduce_umin_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umin_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v13.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v8.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v9.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v8.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v0.h, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v12.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v10.l
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v4.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v10.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l
-; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v3.h, v6.h, v3.h
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v2.h, v3.l, v2.h
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v5.l, v5.h
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v6.l, v7.l
-; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v1.h, v2.l, v1.h
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v6.l
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v6.l, v7.l, v6.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v3.l, v3.l, v5.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v4.h
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v1.l, v1.l, v3.h, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v2.h, v4.l, v7.h
+; GFX11-SDAG-TRUE16-NEXT: v_min_u16 v2.l, v2.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v4.l, v4.h
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v2.h, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v6.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX11-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v2.h
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
@@ -1117,34 +1117,34 @@ define i8 @test_vector_reduce_umin_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v13.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v8.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v9.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v8.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v7.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.h, v0.h, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v12.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v10.l
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v1.l, v4.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v10.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v14.l
-; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v3.h, v6.h, v3.h
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v2.h, v3.l, v2.h
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v5.l, v5.h
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.l, v6.l, v7.l
-; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v1.h, v2.l, v1.h
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v6.l
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v6.l, v7.l, v6.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v3.l, v3.l, v5.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v0.l, v0.l, v4.h
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v1.l, v1.l, v3.h, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v2.h, v4.l, v7.h
+; GFX12-SDAG-TRUE16-NEXT: v_min_u16 v2.l, v2.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v4.l, v4.h
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v0.h, v2.h, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.h, v1.l, v3.l, v6.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v1.h, v1.l
+; GFX12-SDAG-TRUE16-NEXT: v_min3_u16 v0.l, v0.l, v2.l, v2.h
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
index b68654a2f9ff4..9c41fcf877a08 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
@@ -794,23 +794,23 @@ define i8 @test_vector_reduce_xor_v16i8(<16 x i8> %v) {
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_xor_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v5.l, v13.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v9.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v5.l, v7.l, v15.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v6.l, v14.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v7.l, v15.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.l, v10.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, v3.l, v11.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v3.l, v4.l, v12.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v3.l, v3.l, v11.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v3.h, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v8.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v1.h
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.h, v5.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.l, v2.h
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v3.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v3.l
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v2.l
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v3.h
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v0.h, v1.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -865,23 +865,23 @@ define i8 @test_vector_reduce_xor_v16i8(<16 x i8> %v) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v5.l, v13.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v9.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v5.l, v7.l, v15.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v6.l, v14.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v7.l, v15.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.l, v10.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.h, v3.l, v11.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v3.l, v4.l, v12.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v3.l, v3.l, v11.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v3.h, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v8.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v1.h
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v2.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.h, v5.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v2.l, v2.l, v2.h
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.h, v3.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v3.l
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v1.l, v1.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v3.h
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.h, v0.h, v1.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_xor_b16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll b/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
index 146fe11ae3745..d424943ed95b7 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_rebroadcast.ll
@@ -28,9 +28,9 @@ define <2 x i8> @shuffle_v2i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.l, 8, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v2i8_rebroadcast:
@@ -76,11 +76,11 @@ define <4 x i8> @shuffle_v4i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4i8_rebroadcast:
@@ -136,15 +136,15 @@ define <8 x i8> @shuffle_v8i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v8i8_rebroadcast:
@@ -220,23 +220,23 @@ define <16 x i8> @shuffle_v16i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v16i8_rebroadcast:
@@ -352,39 +352,39 @@ define <32 x i8> @shuffle_v32i8_rebroadcast(ptr addrspace(1) %arg0) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v31.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v0.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v0.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v32i8_rebroadcast:
>From 5c6ad7af17d13f459d2e437ee5e8f38b747d6b9a Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Mon, 31 Aug 2026 11:40:48 -0400
Subject: [PATCH 2/3] update test
---
.../GlobalISel/combine-fma-add-fma-mul.ll | 872 +++++++++---------
...llvm.amdgcn.raw.buffer.store.format.f16.ll | 26 +-
.../llvm.amdgcn.raw.buffer.store.ll | 14 +-
.../llvm.amdgcn.raw.tbuffer.store.f16.ll | 268 +++---
...vm.amdgcn.struct.buffer.load.format.f16.ll | 15 +-
...m.amdgcn.struct.buffer.store.format.f16.ll | 38 +-
.../CodeGen/AMDGPU/call-argument-types.ll | 45 +-
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 25 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll | 54 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 364 +++-----
llvm/test/CodeGen/AMDGPU/function-returns.ll | 45 +-
.../AMDGPU/gfx-callable-argument-types.ll | 276 +++---
.../CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll | 52 +-
llvm/test/CodeGen/AMDGPU/llvm.frexp.ll | 48 +-
llvm/test/CodeGen/AMDGPU/usubsat.ll | 131 +--
15 files changed, 1008 insertions(+), 1265 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
index 430bc08a704a4..641f2ef840cc3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 -fp-contract=fast < %s | FileCheck -check-prefix=GFX9-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
-; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -fp-contract=fast < %s | FileCheck -check-prefixes=GFX11-CONTRACT,GFX11-CONTRACT-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX11-DENORM,GFX11-DENORM-FAKE16 %s
@@ -12,12 +12,12 @@
; fadd e, (fma a, b, (fmul c, d)) --> fma a, b, (fma c, d, e)
define float @test_f32_add_mul(float %a, float %b, float %c, float %d, float %e) {
-; GFX9-LABEL: test_f32_add_mul:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_fma_f32 v2, v2, v3, v4
-; GFX9-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_f32_add_mul:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f32 v2, v2, v3, v4
+; GFX9-CONTRACT-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_f32_add_mul:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -27,13 +27,13 @@ define float @test_f32_add_mul(float %a, float %b, float %c, float %d, float %e)
; GFX9-DENORM-NEXT: v_mov_b32_e32 v0, v2
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_f32_add_mul:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_fma_f32 v2, v2, v3, v4
-; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX10-NEXT: v_mov_b32_e32 v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_f32_add_mul:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f32 v2, v2, v3, v4
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_f32_add_mul:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -43,14 +43,14 @@ define float @test_f32_add_mul(float %a, float %b, float %c, float %d, float %e)
; GFX10-DENORM-NEXT: v_mov_b32_e32 v0, v2
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_f32_add_mul:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f32 v2, v2, v3, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX11-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_f32_add_mul:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: v_fma_f32 v2, v2, v3, v4
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX11-CONTRACT-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_f32_add_mul:
; GFX11-DENORM: ; %bb.0: ; %.entry
@@ -68,12 +68,12 @@ define float @test_f32_add_mul(float %a, float %b, float %c, float %d, float %e)
}
define float @test_f32_add_mul_rhs(float %a, float %b, float %c, float %d, float %e) {
-; GFX9-LABEL: test_f32_add_mul_rhs:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_fma_f32 v2, v2, v3, v4
-; GFX9-NEXT: v_fma_f32 v0, v0, v1, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_f32_add_mul_rhs:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f32 v2, v2, v3, v4
+; GFX9-CONTRACT-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_f32_add_mul_rhs:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -83,13 +83,13 @@ define float @test_f32_add_mul_rhs(float %a, float %b, float %c, float %d, float
; GFX9-DENORM-NEXT: v_mov_b32_e32 v0, v2
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_f32_add_mul_rhs:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_fma_f32 v2, v2, v3, v4
-; GFX10-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX10-NEXT: v_mov_b32_e32 v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_f32_add_mul_rhs:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f32 v2, v2, v3, v4
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_f32_add_mul_rhs:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -99,14 +99,14 @@ define float @test_f32_add_mul_rhs(float %a, float %b, float %c, float %d, float
; GFX10-DENORM-NEXT: v_mov_b32_e32 v0, v2
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_f32_add_mul_rhs:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f32 v2, v2, v3, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_fmac_f32_e32 v2, v0, v1
-; GFX11-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_f32_add_mul_rhs:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: v_fma_f32 v2, v2, v3, v4
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT: v_fmac_f32_e32 v2, v0, v1
+; GFX11-CONTRACT-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_f32_add_mul_rhs:
; GFX11-DENORM: ; %bb.0: ; %.entry
@@ -124,12 +124,12 @@ define float @test_f32_add_mul_rhs(float %a, float %b, float %c, float %d, float
}
define half @test_half_add_mul(half %a, half %b, half %c, half %d, half %e) {
-; GFX9-LABEL: test_half_add_mul:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_fma_f16 v2, v2, v3, v4
-; GFX9-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_half_add_mul:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f16 v2, v2, v3, v4
+; GFX9-CONTRACT-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_half_add_mul:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -139,13 +139,13 @@ define half @test_half_add_mul(half %a, half %b, half %c, half %d, half %e) {
; GFX9-DENORM-NEXT: v_mov_b32_e32 v0, v2
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_half_add_mul:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_fma_f16 v2, v2, v3, v4
-; GFX10-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX10-NEXT: v_mov_b32_e32 v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_half_add_mul:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f16 v2, v2, v3, v4
+; GFX10-CONTRACT-NEXT: v_fmac_f16_e32 v2, v0, v1
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_half_add_mul:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -156,23 +156,23 @@ define half @test_half_add_mul(half %a, half %b, half %c, half %d, half %e) {
; GFX10-DENORM-NEXT: v_add_f16_e32 v0, v0, v4
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: test_half_add_mul:
-; GFX11-TRUE16: ; %bb.0: ; %.entry
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_fma_f16 v2.l, v2.l, v3.l, v4.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: test_half_add_mul:
-; GFX11-FAKE16: ; %bb.0: ; %.entry
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_fma_f16 v2, v2, v3, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-TRUE16-LABEL: test_half_add_mul:
+; GFX11-CONTRACT-TRUE16: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-TRUE16-NEXT: v_fma_f16 v0.h, v2.l, v3.l, v4.l
+; GFX11-CONTRACT-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-TRUE16-NEXT: v_fmac_f16_e32 v0.h, v0.l, v1.l
+; GFX11-CONTRACT-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-CONTRACT-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-FAKE16-LABEL: test_half_add_mul:
+; GFX11-CONTRACT-FAKE16: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-FAKE16-NEXT: v_fma_f16 v2, v2, v3, v4
+; GFX11-CONTRACT-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
+; GFX11-CONTRACT-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-TRUE16-LABEL: test_half_add_mul:
; GFX11-DENORM-TRUE16: ; %bb.0: ; %.entry
@@ -201,12 +201,12 @@ define half @test_half_add_mul(half %a, half %b, half %c, half %d, half %e) {
}
define half @test_half_add_mul_rhs(half %a, half %b, half %c, half %d, half %e) {
-; GFX9-LABEL: test_half_add_mul_rhs:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_fma_f16 v2, v2, v3, v4
-; GFX9-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_half_add_mul_rhs:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f16 v2, v2, v3, v4
+; GFX9-CONTRACT-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_half_add_mul_rhs:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -216,13 +216,13 @@ define half @test_half_add_mul_rhs(half %a, half %b, half %c, half %d, half %e)
; GFX9-DENORM-NEXT: v_mov_b32_e32 v0, v2
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_half_add_mul_rhs:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_fma_f16 v2, v2, v3, v4
-; GFX10-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX10-NEXT: v_mov_b32_e32 v0, v2
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_half_add_mul_rhs:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f16 v2, v2, v3, v4
+; GFX10-CONTRACT-NEXT: v_fmac_f16_e32 v2, v0, v1
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v0, v2
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_half_add_mul_rhs:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -233,23 +233,23 @@ define half @test_half_add_mul_rhs(half %a, half %b, half %c, half %d, half %e)
; GFX10-DENORM-NEXT: v_add_f16_e32 v0, v4, v0
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: test_half_add_mul_rhs:
-; GFX11-TRUE16: ; %bb.0: ; %.entry
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_fma_f16 v2.l, v2.l, v3.l, v4.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: test_half_add_mul_rhs:
-; GFX11-FAKE16: ; %bb.0: ; %.entry
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_fma_f16 v2, v2, v3, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-TRUE16-LABEL: test_half_add_mul_rhs:
+; GFX11-CONTRACT-TRUE16: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-TRUE16-NEXT: v_fma_f16 v0.h, v2.l, v3.l, v4.l
+; GFX11-CONTRACT-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-TRUE16-NEXT: v_fmac_f16_e32 v0.h, v0.l, v1.l
+; GFX11-CONTRACT-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-CONTRACT-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-FAKE16-LABEL: test_half_add_mul_rhs:
+; GFX11-CONTRACT-FAKE16: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-FAKE16-NEXT: v_fma_f16 v2, v2, v3, v4
+; GFX11-CONTRACT-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-FAKE16-NEXT: v_fmac_f16_e32 v2, v0, v1
+; GFX11-CONTRACT-FAKE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-TRUE16-LABEL: test_half_add_mul_rhs:
; GFX11-DENORM-TRUE16: ; %bb.0: ; %.entry
@@ -278,12 +278,12 @@ define half @test_half_add_mul_rhs(half %a, half %b, half %c, half %d, half %e)
}
define double @test_double_add_mul(double %a, double %b, double %c, double %d, double %e) {
-; GFX9-LABEL: test_double_add_mul:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
-; GFX9-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_double_add_mul:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_double_add_mul:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -292,12 +292,12 @@ define double @test_double_add_mul(double %a, double %b, double %c, double %d, d
; GFX9-DENORM-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_double_add_mul:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
-; GFX10-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_double_add_mul:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_double_add_mul:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -306,13 +306,13 @@ define double @test_double_add_mul(double %a, double %b, double %c, double %d, d
; GFX10-DENORM-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_double_add_mul:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_double_add_mul:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_double_add_mul:
; GFX11-DENORM: ; %bb.0: ; %.entry
@@ -329,12 +329,12 @@ define double @test_double_add_mul(double %a, double %b, double %c, double %d, d
}
define double @test_double_add_mul_rhs(double %a, double %b, double %c, double %d, double %e) {
-; GFX9-LABEL: test_double_add_mul_rhs:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
-; GFX9-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_double_add_mul_rhs:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_double_add_mul_rhs:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -343,12 +343,12 @@ define double @test_double_add_mul_rhs(double %a, double %b, double %c, double %
; GFX9-DENORM-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_double_add_mul_rhs:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
-; GFX10-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_double_add_mul_rhs:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_double_add_mul_rhs:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -357,13 +357,13 @@ define double @test_double_add_mul_rhs(double %a, double %b, double %c, double %
; GFX10-DENORM-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_double_add_mul_rhs:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_double_add_mul_rhs:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_double_add_mul_rhs:
; GFX11-DENORM: ; %bb.0: ; %.entry
@@ -380,18 +380,18 @@ define double @test_double_add_mul_rhs(double %a, double %b, double %c, double %
}
define <4 x float> @test_v4f32_add_mul(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, <4 x float> %e) {
-; GFX9-LABEL: test_v4f32_add_mul:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_fma_f32 v8, v8, v12, v16
-; GFX9-NEXT: v_fma_f32 v9, v9, v13, v17
-; GFX9-NEXT: v_fma_f32 v10, v10, v14, v18
-; GFX9-NEXT: v_fma_f32 v11, v11, v15, v19
-; GFX9-NEXT: v_fma_f32 v0, v0, v4, v8
-; GFX9-NEXT: v_fma_f32 v1, v1, v5, v9
-; GFX9-NEXT: v_fma_f32 v2, v2, v6, v10
-; GFX9-NEXT: v_fma_f32 v3, v3, v7, v11
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_v4f32_add_mul:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f32 v8, v8, v12, v16
+; GFX9-CONTRACT-NEXT: v_fma_f32 v9, v9, v13, v17
+; GFX9-CONTRACT-NEXT: v_fma_f32 v10, v10, v14, v18
+; GFX9-CONTRACT-NEXT: v_fma_f32 v11, v11, v15, v19
+; GFX9-CONTRACT-NEXT: v_fma_f32 v0, v0, v4, v8
+; GFX9-CONTRACT-NEXT: v_fma_f32 v1, v1, v5, v9
+; GFX9-CONTRACT-NEXT: v_fma_f32 v2, v2, v6, v10
+; GFX9-CONTRACT-NEXT: v_fma_f32 v3, v3, v7, v11
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_v4f32_add_mul:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -410,22 +410,22 @@ define <4 x float> @test_v4f32_add_mul(<4 x float> %a, <4 x float> %b, <4 x floa
; GFX9-DENORM-NEXT: v_mov_b32_e32 v3, v6
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_v4f32_add_mul:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_fma_f32 v8, v8, v12, v16
-; GFX10-NEXT: v_fma_f32 v9, v9, v13, v17
-; GFX10-NEXT: v_fma_f32 v10, v10, v14, v18
-; GFX10-NEXT: v_fma_f32 v11, v11, v15, v19
-; GFX10-NEXT: v_fmac_f32_e32 v8, v0, v4
-; GFX10-NEXT: v_fmac_f32_e32 v9, v1, v5
-; GFX10-NEXT: v_fmac_f32_e32 v10, v2, v6
-; GFX10-NEXT: v_fmac_f32_e32 v11, v3, v7
-; GFX10-NEXT: v_mov_b32_e32 v0, v8
-; GFX10-NEXT: v_mov_b32_e32 v1, v9
-; GFX10-NEXT: v_mov_b32_e32 v2, v10
-; GFX10-NEXT: v_mov_b32_e32 v3, v11
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_v4f32_add_mul:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f32 v8, v8, v12, v16
+; GFX10-CONTRACT-NEXT: v_fma_f32 v9, v9, v13, v17
+; GFX10-CONTRACT-NEXT: v_fma_f32 v10, v10, v14, v18
+; GFX10-CONTRACT-NEXT: v_fma_f32 v11, v11, v15, v19
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v8, v0, v4
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v9, v1, v5
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v10, v2, v6
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v11, v3, v7
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v0, v8
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v1, v9
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v2, v10
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v3, v11
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_v4f32_add_mul:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -444,20 +444,20 @@ define <4 x float> @test_v4f32_add_mul(<4 x float> %a, <4 x float> %b, <4 x floa
; GFX10-DENORM-NEXT: v_mov_b32_e32 v3, v11
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_v4f32_add_mul:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f32 v8, v8, v12, v16
-; GFX11-NEXT: v_fma_f32 v9, v9, v13, v17
-; GFX11-NEXT: v_fma_f32 v10, v10, v14, v18
-; GFX11-NEXT: v_fma_f32 v11, v11, v15, v19
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_fmac_f32 v8, v0, v4 :: v_dual_fmac_f32 v9, v1, v5
-; GFX11-NEXT: v_dual_fmac_f32 v10, v2, v6 :: v_dual_fmac_f32 v11, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
-; GFX11-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_v4f32_add_mul:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: v_fma_f32 v8, v8, v12, v16
+; GFX11-CONTRACT-NEXT: v_fma_f32 v9, v9, v13, v17
+; GFX11-CONTRACT-NEXT: v_fma_f32 v10, v10, v14, v18
+; GFX11-CONTRACT-NEXT: v_fma_f32 v11, v11, v15, v19
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT: v_dual_fmac_f32 v8, v0, v4 :: v_dual_fmac_f32 v9, v1, v5
+; GFX11-CONTRACT-NEXT: v_dual_fmac_f32 v10, v2, v6 :: v_dual_fmac_f32 v11, v3, v7
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-CONTRACT-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_v4f32_add_mul:
; GFX11-DENORM: ; %bb.0: ; %.entry
@@ -481,18 +481,18 @@ define <4 x float> @test_v4f32_add_mul(<4 x float> %a, <4 x float> %b, <4 x floa
}
define <4 x float> @test_v4f32_add_mul_rhs(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, <4 x float> %e) {
-; GFX9-LABEL: test_v4f32_add_mul_rhs:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_fma_f32 v8, v8, v12, v16
-; GFX9-NEXT: v_fma_f32 v9, v9, v13, v17
-; GFX9-NEXT: v_fma_f32 v10, v10, v14, v18
-; GFX9-NEXT: v_fma_f32 v11, v11, v15, v19
-; GFX9-NEXT: v_fma_f32 v0, v0, v4, v8
-; GFX9-NEXT: v_fma_f32 v1, v1, v5, v9
-; GFX9-NEXT: v_fma_f32 v2, v2, v6, v10
-; GFX9-NEXT: v_fma_f32 v3, v3, v7, v11
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_v4f32_add_mul_rhs:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f32 v8, v8, v12, v16
+; GFX9-CONTRACT-NEXT: v_fma_f32 v9, v9, v13, v17
+; GFX9-CONTRACT-NEXT: v_fma_f32 v10, v10, v14, v18
+; GFX9-CONTRACT-NEXT: v_fma_f32 v11, v11, v15, v19
+; GFX9-CONTRACT-NEXT: v_fma_f32 v0, v0, v4, v8
+; GFX9-CONTRACT-NEXT: v_fma_f32 v1, v1, v5, v9
+; GFX9-CONTRACT-NEXT: v_fma_f32 v2, v2, v6, v10
+; GFX9-CONTRACT-NEXT: v_fma_f32 v3, v3, v7, v11
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_v4f32_add_mul_rhs:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -511,22 +511,22 @@ define <4 x float> @test_v4f32_add_mul_rhs(<4 x float> %a, <4 x float> %b, <4 x
; GFX9-DENORM-NEXT: v_mov_b32_e32 v3, v6
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_v4f32_add_mul_rhs:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_fma_f32 v8, v8, v12, v16
-; GFX10-NEXT: v_fma_f32 v9, v9, v13, v17
-; GFX10-NEXT: v_fma_f32 v10, v10, v14, v18
-; GFX10-NEXT: v_fma_f32 v11, v11, v15, v19
-; GFX10-NEXT: v_fmac_f32_e32 v8, v0, v4
-; GFX10-NEXT: v_fmac_f32_e32 v9, v1, v5
-; GFX10-NEXT: v_fmac_f32_e32 v10, v2, v6
-; GFX10-NEXT: v_fmac_f32_e32 v11, v3, v7
-; GFX10-NEXT: v_mov_b32_e32 v0, v8
-; GFX10-NEXT: v_mov_b32_e32 v1, v9
-; GFX10-NEXT: v_mov_b32_e32 v2, v10
-; GFX10-NEXT: v_mov_b32_e32 v3, v11
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_v4f32_add_mul_rhs:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f32 v8, v8, v12, v16
+; GFX10-CONTRACT-NEXT: v_fma_f32 v9, v9, v13, v17
+; GFX10-CONTRACT-NEXT: v_fma_f32 v10, v10, v14, v18
+; GFX10-CONTRACT-NEXT: v_fma_f32 v11, v11, v15, v19
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v8, v0, v4
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v9, v1, v5
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v10, v2, v6
+; GFX10-CONTRACT-NEXT: v_fmac_f32_e32 v11, v3, v7
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v0, v8
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v1, v9
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v2, v10
+; GFX10-CONTRACT-NEXT: v_mov_b32_e32 v3, v11
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_v4f32_add_mul_rhs:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -545,20 +545,20 @@ define <4 x float> @test_v4f32_add_mul_rhs(<4 x float> %a, <4 x float> %b, <4 x
; GFX10-DENORM-NEXT: v_mov_b32_e32 v3, v11
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_v4f32_add_mul_rhs:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_fma_f32 v8, v8, v12, v16
-; GFX11-NEXT: v_fma_f32 v9, v9, v13, v17
-; GFX11-NEXT: v_fma_f32 v10, v10, v14, v18
-; GFX11-NEXT: v_fma_f32 v11, v11, v15, v19
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_fmac_f32 v8, v0, v4 :: v_dual_fmac_f32 v9, v1, v5
-; GFX11-NEXT: v_dual_fmac_f32 v10, v2, v6 :: v_dual_fmac_f32 v11, v3, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
-; GFX11-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_v4f32_add_mul_rhs:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: v_fma_f32 v8, v8, v12, v16
+; GFX11-CONTRACT-NEXT: v_fma_f32 v9, v9, v13, v17
+; GFX11-CONTRACT-NEXT: v_fma_f32 v10, v10, v14, v18
+; GFX11-CONTRACT-NEXT: v_fma_f32 v11, v11, v15, v19
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT: v_dual_fmac_f32 v8, v0, v4 :: v_dual_fmac_f32 v9, v1, v5
+; GFX11-CONTRACT-NEXT: v_dual_fmac_f32 v10, v2, v6 :: v_dual_fmac_f32 v11, v3, v7
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-CONTRACT-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_v4f32_add_mul_rhs:
; GFX11-DENORM: ; %bb.0: ; %.entry
@@ -582,14 +582,14 @@ define <4 x float> @test_v4f32_add_mul_rhs(<4 x float> %a, <4 x float> %b, <4 x
}
define <4 x half> @test_f16_add_mul(<4 x half> %a, <4 x half> %b, <4 x half> %c, <4 x half> %d, <4 x half> %e) {
-; GFX9-LABEL: test_f16_add_mul:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_fma_f16 v4, v4, v6, v8
-; GFX9-NEXT: v_pk_fma_f16 v5, v5, v7, v9
-; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_f16_add_mul:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_pk_fma_f16 v4, v4, v6, v8
+; GFX9-CONTRACT-NEXT: v_pk_fma_f16 v5, v5, v7, v9
+; GFX9-CONTRACT-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX9-CONTRACT-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_f16_add_mul:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -604,14 +604,14 @@ define <4 x half> @test_f16_add_mul(<4 x half> %a, <4 x half> %b, <4 x half> %c,
; GFX9-DENORM-NEXT: v_pk_add_f16 v1, v1, v9
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_f16_add_mul:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_fma_f16 v4, v4, v6, v8
-; GFX10-NEXT: v_pk_fma_f16 v5, v5, v7, v9
-; GFX10-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX10-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_f16_add_mul:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_pk_fma_f16 v4, v4, v6, v8
+; GFX10-CONTRACT-NEXT: v_pk_fma_f16 v5, v5, v7, v9
+; GFX10-CONTRACT-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX10-CONTRACT-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_f16_add_mul:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -626,15 +626,15 @@ define <4 x half> @test_f16_add_mul(<4 x half> %a, <4 x half> %b, <4 x half> %c,
; GFX10-DENORM-NEXT: v_pk_add_f16 v1, v1, v9
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_f16_add_mul:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_fma_f16 v4, v4, v6, v8
-; GFX11-NEXT: v_pk_fma_f16 v5, v5, v7, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX11-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_f16_add_mul:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: v_pk_fma_f16 v4, v4, v6, v8
+; GFX11-CONTRACT-NEXT: v_pk_fma_f16 v5, v5, v7, v9
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX11-CONTRACT-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_f16_add_mul:
; GFX11-DENORM: ; %bb.0: ; %.entry
@@ -658,14 +658,14 @@ define <4 x half> @test_f16_add_mul(<4 x half> %a, <4 x half> %b, <4 x half> %c,
}
define <4 x half> @test_f16_add_mul_rhs(<4 x half> %a, <4 x half> %b, <4 x half> %c, <4 x half> %d, <4 x half> %e) {
-; GFX9-LABEL: test_f16_add_mul_rhs:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_fma_f16 v4, v4, v6, v8
-; GFX9-NEXT: v_pk_fma_f16 v5, v5, v7, v9
-; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_f16_add_mul_rhs:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: v_pk_fma_f16 v4, v4, v6, v8
+; GFX9-CONTRACT-NEXT: v_pk_fma_f16 v5, v5, v7, v9
+; GFX9-CONTRACT-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX9-CONTRACT-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_f16_add_mul_rhs:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -680,14 +680,14 @@ define <4 x half> @test_f16_add_mul_rhs(<4 x half> %a, <4 x half> %b, <4 x half>
; GFX9-DENORM-NEXT: v_pk_add_f16 v1, v9, v1
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_f16_add_mul_rhs:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_fma_f16 v4, v4, v6, v8
-; GFX10-NEXT: v_pk_fma_f16 v5, v5, v7, v9
-; GFX10-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX10-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_f16_add_mul_rhs:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: v_pk_fma_f16 v4, v4, v6, v8
+; GFX10-CONTRACT-NEXT: v_pk_fma_f16 v5, v5, v7, v9
+; GFX10-CONTRACT-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX10-CONTRACT-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_f16_add_mul_rhs:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -702,15 +702,15 @@ define <4 x half> @test_f16_add_mul_rhs(<4 x half> %a, <4 x half> %b, <4 x half>
; GFX10-DENORM-NEXT: v_pk_add_f16 v1, v9, v1
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_f16_add_mul_rhs:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_pk_fma_f16 v4, v4, v6, v8
-; GFX11-NEXT: v_pk_fma_f16 v5, v5, v7, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX11-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_f16_add_mul_rhs:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: v_pk_fma_f16 v4, v4, v6, v8
+; GFX11-CONTRACT-NEXT: v_pk_fma_f16 v5, v5, v7, v9
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX11-CONTRACT-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_f16_add_mul_rhs:
; GFX11-DENORM: ; %bb.0: ; %.entry
@@ -734,31 +734,31 @@ define <4 x half> @test_f16_add_mul_rhs(<4 x half> %a, <4 x half> %b, <4 x half>
}
define <4 x double> @test_f64_add_mul(<4 x double> %a, <4 x double> %b, <4 x double> %c, <4 x double> %d, <4 x double> %e) {
-; GFX9-LABEL: test_f64_add_mul:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[31:32]
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:12
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:16
-; GFX9-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[24:25]
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:24
-; GFX9-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[24:25]
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:32
-; GFX9-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[24:25]
-; GFX9-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_f64_add_mul:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
+; GFX9-CONTRACT-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[31:32]
+; GFX9-CONTRACT-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:12
+; GFX9-CONTRACT-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:16
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[24:25]
+; GFX9-CONTRACT-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:20
+; GFX9-CONTRACT-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:24
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[24:25]
+; GFX9-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-CONTRACT-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:28
+; GFX9-CONTRACT-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:32
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[24:25]
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_f64_add_mul:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -786,32 +786,32 @@ define <4 x double> @test_f64_add_mul(<4 x double> %a, <4 x double> %b, <4 x dou
; GFX9-DENORM-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_f64_add_mul:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_clause 0x8
-; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
-; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
-; GFX10-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
-; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
-; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
-; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
-; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
-; GFX10-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
-; GFX10-NEXT: s_waitcnt vmcnt(6)
-; GFX10-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX10-NEXT: s_waitcnt vmcnt(4)
-; GFX10-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX10-NEXT: s_waitcnt vmcnt(2)
-; GFX10-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
-; GFX10-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
-; GFX10-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
-; GFX10-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
-; GFX10-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_f64_add_mul:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: s_clause 0x8
+; GFX10-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX10-CONTRACT-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX10-CONTRACT-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GFX10-CONTRACT-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
+; GFX10-CONTRACT-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GFX10-CONTRACT-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
+; GFX10-CONTRACT-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
+; GFX10-CONTRACT-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
+; GFX10-CONTRACT-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(6)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(4)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(2)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_f64_add_mul:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -840,34 +840,34 @@ define <4 x double> @test_f64_add_mul(<4 x double> %a, <4 x double> %b, <4 x dou
; GFX10-DENORM-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_f64_add_mul:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_clause 0x8
-; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:12
-; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:16
-; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:20
-; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:24
-; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:28
-; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:32
-; GFX11-NEXT: s_waitcnt vmcnt(6)
-; GFX11-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX11-NEXT: s_waitcnt vmcnt(4)
-; GFX11-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX11-NEXT: s_waitcnt vmcnt(2)
-; GFX11-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
-; GFX11-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
-; GFX11-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_f64_add_mul:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: s_clause 0x8
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v32, off, s32 offset:4
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v33, off, s32 offset:8
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v34, off, s32 offset:12
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v35, off, s32 offset:16
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v36, off, s32 offset:20
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v37, off, s32 offset:24
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v38, off, s32 offset:28
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v39, off, s32 offset:32
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(6)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(4)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(2)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_f64_add_mul:
; GFX11-DENORM: ; %bb.0: ; %.entry
@@ -905,31 +905,31 @@ define <4 x double> @test_f64_add_mul(<4 x double> %a, <4 x double> %b, <4 x dou
}
define <4 x double> @test_f64_add_mul_rhs(<4 x double> %a, <4 x double> %b, <4 x double> %c, <4 x double> %d, <4 x double> %e) {
-; GFX9-LABEL: test_f64_add_mul_rhs:
-; GFX9: ; %bb.0: ; %.entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
-; GFX9-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[31:32]
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:12
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:16
-; GFX9-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[24:25]
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:20
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:24
-; GFX9-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[24:25]
-; GFX9-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX9-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:28
-; GFX9-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:32
-; GFX9-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[24:25]
-; GFX9-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX9-CONTRACT-LABEL: test_f64_add_mul_rhs:
+; GFX9-CONTRACT: ; %bb.0: ; %.entry
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:4
+; GFX9-CONTRACT-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:8
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[31:32]
+; GFX9-CONTRACT-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:12
+; GFX9-CONTRACT-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:16
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[24:25]
+; GFX9-CONTRACT-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:20
+; GFX9-CONTRACT-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:24
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[24:25]
+; GFX9-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX9-CONTRACT-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:28
+; GFX9-CONTRACT-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:32
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX9-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[24:25]
+; GFX9-CONTRACT-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX9-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-DENORM-LABEL: test_f64_add_mul_rhs:
; GFX9-DENORM: ; %bb.0: ; %.entry
@@ -957,32 +957,32 @@ define <4 x double> @test_f64_add_mul_rhs(<4 x double> %a, <4 x double> %b, <4 x
; GFX9-DENORM-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
; GFX9-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_f64_add_mul_rhs:
-; GFX10: ; %bb.0: ; %.entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: s_clause 0x8
-; GFX10-NEXT: buffer_load_dword v31, off, s[0:3], s32
-; GFX10-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
-; GFX10-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
-; GFX10-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
-; GFX10-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
-; GFX10-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
-; GFX10-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
-; GFX10-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
-; GFX10-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
-; GFX10-NEXT: s_waitcnt vmcnt(6)
-; GFX10-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX10-NEXT: s_waitcnt vmcnt(4)
-; GFX10-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX10-NEXT: s_waitcnt vmcnt(2)
-; GFX10-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
-; GFX10-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
-; GFX10-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
-; GFX10-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
-; GFX10-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX10-CONTRACT-LABEL: test_f64_add_mul_rhs:
+; GFX10-CONTRACT: ; %bb.0: ; %.entry
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-CONTRACT-NEXT: s_clause 0x8
+; GFX10-CONTRACT-NEXT: buffer_load_dword v31, off, s[0:3], s32
+; GFX10-CONTRACT-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:4
+; GFX10-CONTRACT-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:8
+; GFX10-CONTRACT-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:12
+; GFX10-CONTRACT-NEXT: buffer_load_dword v35, off, s[0:3], s32 offset:16
+; GFX10-CONTRACT-NEXT: buffer_load_dword v36, off, s[0:3], s32 offset:20
+; GFX10-CONTRACT-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:24
+; GFX10-CONTRACT-NEXT: buffer_load_dword v38, off, s[0:3], s32 offset:28
+; GFX10-CONTRACT-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:32
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(6)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(4)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(2)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
+; GFX10-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX10-CONTRACT-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX10-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-DENORM-LABEL: test_f64_add_mul_rhs:
; GFX10-DENORM: ; %bb.0: ; %.entry
@@ -1011,34 +1011,34 @@ define <4 x double> @test_f64_add_mul_rhs(<4 x double> %a, <4 x double> %b, <4 x
; GFX10-DENORM-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
; GFX10-DENORM-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_f64_add_mul_rhs:
-; GFX11: ; %bb.0: ; %.entry
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_clause 0x8
-; GFX11-NEXT: scratch_load_b32 v31, off, s32
-; GFX11-NEXT: scratch_load_b32 v32, off, s32 offset:4
-; GFX11-NEXT: scratch_load_b32 v33, off, s32 offset:8
-; GFX11-NEXT: scratch_load_b32 v34, off, s32 offset:12
-; GFX11-NEXT: scratch_load_b32 v35, off, s32 offset:16
-; GFX11-NEXT: scratch_load_b32 v36, off, s32 offset:20
-; GFX11-NEXT: scratch_load_b32 v37, off, s32 offset:24
-; GFX11-NEXT: scratch_load_b32 v38, off, s32 offset:28
-; GFX11-NEXT: scratch_load_b32 v39, off, s32 offset:32
-; GFX11-NEXT: s_waitcnt vmcnt(6)
-; GFX11-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
-; GFX11-NEXT: s_waitcnt vmcnt(4)
-; GFX11-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
-; GFX11-NEXT: s_waitcnt vmcnt(2)
-; GFX11-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
-; GFX11-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
-; GFX11-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-CONTRACT-LABEL: test_f64_add_mul_rhs:
+; GFX11-CONTRACT: ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT: s_clause 0x8
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v31, off, s32
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v32, off, s32 offset:4
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v33, off, s32 offset:8
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v34, off, s32 offset:12
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v35, off, s32 offset:16
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v36, off, s32 offset:20
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v37, off, s32 offset:24
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v38, off, s32 offset:28
+; GFX11-CONTRACT-NEXT: scratch_load_b32 v39, off, s32 offset:32
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(6)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(4)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(2)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
+; GFX11-CONTRACT-NEXT: s_waitcnt vmcnt(0)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX11-CONTRACT-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX11-CONTRACT-NEXT: v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX11-CONTRACT-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-DENORM-LABEL: test_f64_add_mul_rhs:
; GFX11-DENORM: ; %bb.0: ; %.entry
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
index e355992416d1f..ec4239b164dd3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -36,19 +36,19 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
;
; GFX12-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY4]].lo16
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY4]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -85,18 +85,18 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_409
;
; GFX12-LABEL: name: raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_4095__sgpr_soffset_f16
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY4]].lo16
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFSET_exact [[COPY7]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY4]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFSET_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
index a9ada36d4b2ff..c4658df027377 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
@@ -960,19 +960,19 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
;
; GFX12-LABEL: name: raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr_soffset_f16
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_16 = COPY [[COPY4]].lo16
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
- ; GFX12-NEXT: BUFFER_STORE_SHORT_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY4]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: BUFFER_STORE_SHORT_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
index 4225ab16bac30..051ed6648d01c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
@@ -36,19 +36,19 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@@ -324,41 +324,41 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE1]], [[COPY7]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE2]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -478,44 +478,44 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6
+ ; GFX12-NEXT: liveins: $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -637,45 +637,45 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soff
; GFX12-LABEL: name: raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soffset
; GFX12: bb.1 (%ir-block.0):
; GFX12-NEXT: successors: %bb.2(0x80000000)
- ; GFX12-NEXT: liveins: $sgpr2, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+ ; GFX12-NEXT: liveins: $sgpr2, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr3
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr4
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY6]]
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
; GFX12-NEXT: successors: %bb.3(0x80000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY5]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
- ; GFX12-NEXT: [[COPY13:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY13]], [[COPY11]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
- ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY7]], implicit $exec
+ ; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[S_AND_B32_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY9]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -723,19 +723,19 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_glc
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
ret void
@@ -774,19 +774,19 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
ret void
@@ -825,19 +825,19 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_slc_glc
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
ret void
@@ -876,19 +876,19 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
;
; GFX12-LABEL: name: raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_dlc
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY8]], [[COPY6]], [[REG_SEQUENCE]], [[COPY7]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
index 1588f254b801d..cf71b26c5bad4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
@@ -52,8 +52,9 @@ define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vg
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call half @llvm.amdgcn.struct.buffer.load.format.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret half %val
}
@@ -455,8 +456,9 @@ define amdgpu_ps half @struct_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vg
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 4095, 0, 0, implicit $exec :: (dereferenceable load (f16), align 1, addrspace 8)
- ; GFX12-NEXT: $vgpr0 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%voffset = add i32 %voffset.base, 4095
%val = call half @llvm.amdgcn.struct.buffer.load.format.f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret half %val
@@ -512,9 +514,8 @@ define amdgpu_ps half @struct_buffer_load_format_i16__sgpr_rsrc__vgpr_vindex__vg
; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
; GFX12-NEXT: [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (i16), align 1, addrspace 8)
; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_16 = COPY [[BUFFER_LOAD_FORMAT_D16_X_VBUFFER_BOTHEN]].lo16
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY7]]
- ; GFX12-NEXT: $vgpr0 = COPY [[COPY8]]
- ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
+ ; GFX12-NEXT: $vgpr0_lo16 = COPY [[COPY7]]
+ ; GFX12-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0_lo16
%val = call i16 @llvm.amdgcn.struct.buffer.load.format.i16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
%fval = bitcast i16 %val to half
ret half %fval
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
index 8473cbb2ddceb..d23b45c749352 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
@@ -40,21 +40,21 @@ define amdgpu_ps void @struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr
;
; GFX12-LABEL: name: struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr2, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
- ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[COPY]].lo16
- ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr3
- ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr4
- ; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr5
- ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY2]], %subreg.sub0, [[COPY3]], %subreg.sub1, [[COPY4]], %subreg.sub2, [[COPY5]], %subreg.sub3
- ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr1
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY $vgpr2
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY6]], %subreg.sub0, [[COPY7]], %subreg.sub1
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY9]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY8]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
+ ; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; GFX12-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -455,9 +455,9 @@ define amdgpu_ps void @struct_buffer_store_format_i16__vgpr_val__sgpr_rsrc__vgpr
;
; GFX12-LABEL: name: struct_buffer_store_format_i16__vgpr_val__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
; GFX12: bb.1 (%ir-block.0):
- ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1, $vgpr2
+ ; GFX12-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr1, $vgpr2, $vgpr0_lo16
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr3
; GFX12-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr4
@@ -466,8 +466,10 @@ define amdgpu_ps void @struct_buffer_store_format_i16__vgpr_val__sgpr_rsrc__vgpr
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (i16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (i16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.buffer.store.format.i16(i16 %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index 65218bec1b145..ad9f317c8842b 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -6476,36 +6476,25 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
; GFX11-TRUE16-NEXT: s_addc_u32 s3, s3, external_void_func_v16i8 at rel32@hi+12
; GFX11-TRUE16-NEXT: s_mov_b32 s32, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
+; GFX11-TRUE16-NEXT: buffer_load_b128 v[16:19], off, s[4:7], 0
; GFX11-TRUE16-NEXT: s_mov_b64 s[6:7], s[0:1]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[2:3]
; GFX11-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index e16142458f549..19c5213956606 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -265,23 +265,14 @@ define i16 @s_test_copysign_f16_10_mag(half inreg %sign) {
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: s_test_copysign_f16_10_mag:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, 0x4900
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: s_test_copysign_f16_10_mag:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 0x4900
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: s_test_copysign_f16_10_mag:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, 0x4900
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.copysign.f16(half 10.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
index 5d35d59a1a315..a2e4f3bc150fb 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
@@ -369,47 +369,19 @@ define i8 @test_s_unsigned_i8_f32(float inreg %f) nounwind {
; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-FAKE16-LABEL: test_s_unsigned_i8_f32:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_min_u32 s0, s0, 0xff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-TRUE16-LABEL: test_s_unsigned_i8_f32:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_min_u32 s0, s0, 0xff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-GI-LABEL: test_s_unsigned_i8_f32:
-; GFX12-GI: ; %bb.0:
-; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GI-NEXT: s_wait_expcnt 0x0
-; GFX12-GI-NEXT: s_wait_samplecnt 0x0
-; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xff
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: test_s_unsigned_i8_f32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_min_u32 s0, s0, 0xff
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%x = call i8 @llvm.fptoui.sat.i8.f32(float %f)
ret i8 %x
}
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 35bcf309f8ca0..3a808b2f77441 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -1900,59 +1900,31 @@ define <2 x i8> @test_s_unsigned_v2f64_v2i8(<2 x double> inreg %f) {
; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: test_s_unsigned_v2f64_v2i8:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
-; GFX11-FAKE16-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
-; GFX11-FAKE16-NEXT: v_min_u32_e32 v1, 0xff, v0
-; GFX11-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: test_s_unsigned_v2f64_v2i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
-; GFX11-TRUE16-NEXT: v_cvt_u32_f64_e32 v1, s[0:1]
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: test_s_unsigned_v2f64_v2i8:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
-; GFX12-FAKE16-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
-; GFX12-FAKE16-NEXT: v_min_u32_e32 v1, 0xff, v0
-; GFX12-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v2
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX11-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
+; GFX11-NEXT: v_min_u32_e32 v1, 0xff, v0
+; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 8, v1
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-TRUE16-LABEL: test_s_unsigned_v2f64_v2i8:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
-; GFX12-TRUE16-NEXT: v_cvt_u32_f64_e32 v1, s[0:1]
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-ISEL-LABEL: test_s_unsigned_v2f64_v2i8:
+; GFX12-ISEL: ; %bb.0:
+; GFX12-ISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-ISEL-NEXT: s_wait_expcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_samplecnt 0x0
+; GFX12-ISEL-NEXT: s_wait_bvhcnt 0x0
+; GFX12-ISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v0, s[2:3]
+; GFX12-ISEL-NEXT: v_cvt_u32_f64_e32 v2, s[0:1]
+; GFX12-ISEL-NEXT: v_min_u32_e32 v1, 0xff, v0
+; GFX12-ISEL-NEXT: v_min_u32_e32 v0, 0xff, v2
+; GFX12-ISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v1
+; GFX12-ISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX12-ISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v2f64_v2i8:
; GFX12-GI: ; %bb.0:
@@ -3171,22 +3143,20 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX11-TRUE16-LABEL: test_s_unsigned_v4f16_v4i1:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 1, v2
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 1, v1
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 1, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 1, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v4f16_v4i1:
@@ -3220,24 +3190,21 @@ define <4 x i1> @test_s_unsigned_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
-; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 1, v2
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 1, v1
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 1, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 1, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v4f16_v4i1:
@@ -3397,19 +3364,17 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v2
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v1, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v1, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v4f16_v4i8:
@@ -3465,19 +3430,17 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v2
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v1, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v2, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v1, v3
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v4f16_v4i8:
@@ -5262,9 +5225,9 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s5
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s2
@@ -5274,17 +5237,16 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: v_min_u32_e32 v5, 1, v1
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v8, 1, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 1, v6
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v6, 1, v4
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 1, v8
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v8f16_v8i1:
@@ -5342,9 +5304,9 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s5
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s2
@@ -5355,17 +5317,16 @@ define <8 x i1> @test_s_unsigned_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s4
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff, v6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v8, 1, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 1, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 1, v6
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v6, 1, v4
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 1, v8
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 1, v1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 1, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v8.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v8f16_v8i1:
@@ -5632,56 +5593,51 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s4
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v5.l, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v7, 0xff, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, v1, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v3
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v2, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v7
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v5
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v3, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v1, v0
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v7, v8
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v3, v11
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v1, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[9:10], 24, v[7:8]
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v8f16_v8i8:
@@ -5749,58 +5705,54 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
-; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s4
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
+; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v5.l, s1
-; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v7, 0xff, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v7, v1, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v5, v3
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s0
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v6
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v2, v4
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v7
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v5
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 8, v4
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v6
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, v3, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v1, v2, v8
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v8
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v9
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 8, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[0:1]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v7.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v5.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v10.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v1, v0
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v6
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v7, v8
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v5, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, v3, v11
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v1, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[9:10], 24, v[7:8]
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v8f16_v8i8:
@@ -6770,49 +6722,26 @@ define <4 x i8> @test_s_unsigned_v4f32_v4i8(<4 x float> inreg %f) {
; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: test_s_unsigned_v4f32_v4i8:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v0, s3
-; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v1, s2
-; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v2, s1
-; GFX11-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_min_u32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_min_u32_e32 v3, 0xff, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v1, v0
-; GFX11-FAKE16-NEXT: v_cvt_u32_f32_e32 v0, s0
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 8, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX11-FAKE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v1, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: test_s_unsigned_v4f32_v4i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, s3
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v1, s2
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v2, s1
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v2, 0xff, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v1, v0
-; GFX11-TRUE16-NEXT: v_cvt_u32_f32_e32 v0, s0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v2, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: test_s_unsigned_v4f32_v4i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, s3
+; GFX11-NEXT: v_cvt_u32_f32_e32 v1, s2
+; GFX11-NEXT: v_cvt_u32_f32_e32 v2, s1
+; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX11-NEXT: v_min_u32_e32 v3, 0xff, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX11-NEXT: v_or_b32_e32 v2, v1, v0
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, s0
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-NEXT: v_min_u32_e32 v0, 0xff, v0
+; GFX11-NEXT: v_or_b32_e32 v4, v1, v3
+; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v4
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v4f32_v4i8:
; GFX12-FAKE16: ; %bb.0:
@@ -6874,14 +6803,13 @@ define <4 x i8> @test_s_unsigned_v4f32_v4i8(<4 x float> inreg %f) {
; GFX12-TRUE16-NEXT: s_or_b32 s0, s0, s1
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s1, s1, s3
-; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 24
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-TRUE16-NEXT: s_lshr_b32 s0, s1, 8
+; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s3, 24
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, s1
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, s3
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, s1
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v4f32_v4i8:
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index 66a27bf6258c4..019e49a823aa2 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -1113,35 +1113,24 @@ define <16 x i8> @v16i8_func_void() #0 {
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0
+; GFX11-TRUE16-NEXT: buffer_load_b128 v[16:19], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v16i8_func_void:
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index 14212cb73727b..25d1f81ef8c1d 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -3896,21 +3896,16 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v8i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v8i8 at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[8:9], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
@@ -4160,67 +4155,45 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8() #0 {
; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 0
; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, 16 :: v_dual_mov_b32 v5, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v32i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v32i8 at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v[0:1], off
-; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v[4:5], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[32:35], v[0:1], off
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 16 :: v_dual_mov_b32 v1, 0
+; GFX11-TRUE16-NEXT: global_load_b128 v[36:39], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v35
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v37.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v39
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v39.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
@@ -5521,23 +5494,18 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v8i8_ret at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v8i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[40:41], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[8:9], v[40:41], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
@@ -5908,64 +5876,43 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v42, 16 :: v_dual_mov_b32 v43, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v[40:41], off
-; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v[42:43], off
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: global_load_b128 v[32:35], v[40:41], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[36:39], v[42:43], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v35
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v37.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v39
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v39.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
@@ -9656,37 +9603,26 @@ define amdgpu_gfx void @test_call_external_void_func_v16i8() #0 {
; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v0, s[0:1]
+; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v0, s[0:1]
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v16i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v16i8 at abs32@lo
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v18.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
index 736b58aa04df8..a3ddbdcdcdd50 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
@@ -526,15 +526,13 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) {
; GISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-NEXT: {{ $}}
; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY]]
- ; GISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
- ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY3]]
+ ; GISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY]]
+ ; GISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY1]]
; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
- ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -561,14 +559,12 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) {
; DAGISEL-GFX11-NEXT: {{ $}}
; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY3]]
- ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]]
- ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY]]
+ ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -598,15 +594,13 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %
; GISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-NEXT: {{ $}}
; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY]]
- ; GISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[COPY2]]
- ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY3]]
+ ; GISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY]]
+ ; GISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY1]]
; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
- ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -633,14 +627,12 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %
; DAGISEL-GFX11-NEXT: {{ $}}
; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY3]]
- ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]]
- ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY]]
+ ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -670,13 +662,13 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) {
; GISEL-GFX11-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-NEXT: {{ $}}
; GISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
+ ; GISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
- ; GISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY]]
- ; GISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY1]]
+ ; GISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY]]
+ ; GISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY1]]
; GISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def $scc
; GISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (p0) from got, addrspace 4)
- ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; GISEL-GFX11-NEXT: $sgpr30_sgpr31 = noconvergent SI_CALL [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; GISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc, implicit-def $sgpr32, implicit $sgpr32
; GISEL-GFX11-NEXT: S_ENDPGM 0
;
@@ -703,14 +695,12 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) {
; DAGISEL-GFX11-NEXT: {{ $}}
; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
- ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
- ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; DAGISEL-GFX11-NEXT: $vgpr0 = COPY [[COPY3]]
- ; DAGISEL-GFX11-NEXT: $vgpr1 = COPY [[COPY]]
- ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0, implicit $vgpr1
+ ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY]]
+ ; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: S_ENDPGM 0
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
index 1eebc6f974866..a6e1822d50499 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
@@ -107,13 +107,10 @@ define { half, i32 } @test_frexp_f16_i32(half %a) {
; GFX11-GISEL-TRUE16-LABEL: test_frexp_f16_i32:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v1.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v0.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-GISEL-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX11-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
+; GFX11-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.l, v0.l
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-GISEL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_frexp_f16_i32:
@@ -132,13 +129,10 @@ define { half, i32 } @test_frexp_f16_i32(half %a) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v1.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v0.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-GISEL-TRUE16-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX12-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
+; GFX12-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.l, v0.l
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-GISEL-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 16
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_frexp_f16_i32:
@@ -980,10 +974,10 @@ define { half, i16 } @test_frexp_f16_i16(half %a) {
; GFX11-SDAG-TRUE16-LABEL: test_frexp_f16_i16:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.l, v0.h
-; GFX11-SDAG-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.h
+; GFX11-SDAG-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.h, v0.l
+; GFX11-SDAG-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_frexp_f16_i16:
@@ -1002,10 +996,10 @@ define { half, i16 } @test_frexp_f16_i16(half %a) {
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.l, v0.h
-; GFX12-SDAG-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.h, v0.l
+; GFX12-SDAG-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_frexp_f16_i16:
@@ -1037,11 +1031,10 @@ define { half, i16 } @test_frexp_f16_i16(half %a) {
; GFX11-GISEL-TRUE16-LABEL: test_frexp_f16_i16:
; GFX11-GISEL-TRUE16: ; %bb.0:
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v1.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.h, v0.l
; GFX11-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
-; GFX11-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_frexp_f16_i16:
@@ -1060,11 +1053,10 @@ define { half, i16 } @test_frexp_f16_i16(half %a) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v1.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_frexp_mant_f16_e32 v0.h, v0.l
; GFX12-GISEL-TRUE16-NEXT: v_frexp_exp_i16_f16_e32 v1.l, v0.l
-; GFX12-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v2
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_frexp_f16_i16:
diff --git a/llvm/test/CodeGen/AMDGPU/usubsat.ll b/llvm/test/CodeGen/AMDGPU/usubsat.ll
index fc8b4a1b78694..8ea1c4fa0adc7 100644
--- a/llvm/test/CodeGen/AMDGPU/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/usubsat.ll
@@ -47,25 +47,15 @@ define i8 @s_usubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: s_usubsat_i8:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: s_usubsat_i8:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, s1
-; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, s1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: s_usubsat_i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_max_u32 s0, s0, s1
+; GFX11-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
ret i8 %result
}
@@ -160,25 +150,15 @@ define i16 @s_usubsat_i16(i16 inreg %lhs, i16 inreg %rhs) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: s_usubsat_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: s_usubsat_i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s1, 0xffff, s1
-; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, s1
-; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, s1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: s_usubsat_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-NEXT: s_max_u32 s0, s0, s1
+; GFX11-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%result = call i16 @llvm.usub.sat.i16(i16 %lhs, i16 %rhs)
ret i16 %result
}
@@ -264,23 +244,14 @@ define i16 @uniform_usubsat_as_bithack_i16(i16 inreg %x) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: uniform_usubsat_as_bithack_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, 0x8000
-; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, 0x8000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: uniform_usubsat_as_bithack_i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: uniform_usubsat_as_bithack_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-NEXT: s_max_u32 s0, s0, 0x8000
+; GFX11-NEXT: s_sub_i32 s0, s0, 0x8000
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%signsplat = ashr i16 %x, 15
%flipsign = xor i16 %x, 32768
%result = and i16 %signsplat, %flipsign
@@ -372,23 +343,14 @@ define i16 @uniform_usubsat_as_bithack2_i16(i16 inreg %x) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: uniform_usubsat_as_bithack2_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, 0x8000
-; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, 0x8000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: uniform_usubsat_as_bithack2_i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: uniform_usubsat_as_bithack2_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-NEXT: s_max_u32 s0, s0, 0x8000
+; GFX11-NEXT: s_sub_i32 s0, s0, 0x8000
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%signsplat = ashr i16 %x, 15
%flipsign = add i16 %x, 32768
%result = and i16 %signsplat, %flipsign
@@ -480,23 +442,14 @@ define i16 @uniform_usubsat_as_bithack_commute_i16(i16 inreg %x) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: uniform_usubsat_as_bithack_commute_i16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-TRUE16-NEXT: s_max_u32 s0, s0, 0x8000
-; GFX11-TRUE16-NEXT: s_sub_i32 s0, s0, 0x8000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: uniform_usubsat_as_bithack_commute_i16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, 0x8000
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: uniform_usubsat_as_bithack_commute_i16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX11-NEXT: s_max_u32 s0, s0, 0x8000
+; GFX11-NEXT: s_sub_i32 s0, s0, 0x8000
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%signsplat = ashr i16 %x, 15
%flipsign = add i16 %x, 32768
%result = and i16 %flipsign, %signsplat
>From 088bf35842a81d79bc27b51f3317f161fcd3f372 Mon Sep 17 00:00:00 2001
From: guochen2 <guochen2 at amd.com>
Date: Wed, 9 Sep 2026 13:49:29 -0400
Subject: [PATCH 3/3] rebase and update test
---
.../CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll | 4 +
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 5 +-
...llvm.amdgcn.raw.buffer.store.format.f16.ll | 10 +-
.../llvm.amdgcn.raw.buffer.store.ll | 5 +-
.../llvm.amdgcn.raw.tbuffer.store.f16.ll | 84 +-
...m.amdgcn.struct.buffer.store.format.f16.ll | 7 +-
.../AMDGPU/GlobalISel/load-constant.96.ll | 74 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll | 3641 +++++++------
.../CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll | 116 +-
.../CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll | 4714 +++++++++--------
.../CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll | 1562 +++---
.../CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll | 1162 ++--
.../CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll | 2605 +++++----
.../CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll | 1953 ++++---
llvm/test/CodeGen/AMDGPU/bf16.ll | 375 +-
.../buffer-fat-pointer-atomicrmw-fadd.ll | 642 +--
.../buffer-fat-pointer-atomicrmw-fmax.ll | 642 +--
.../buffer-fat-pointer-atomicrmw-fmin.ll | 642 +--
.../CodeGen/AMDGPU/call-argument-types.ll | 36 +-
.../CodeGen/AMDGPU/calling-conventions.ll | 34 +-
.../CodeGen/AMDGPU/dagcombine-fmul-sel.ll | 120 +-
llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll | 205 +-
.../CodeGen/AMDGPU/flat-atomicrmw-fadd.ll | 1637 ++----
.../CodeGen/AMDGPU/flat-atomicrmw-fmax.ll | 1649 ++----
.../CodeGen/AMDGPU/flat-atomicrmw-fmin.ll | 1649 ++----
.../CodeGen/AMDGPU/flat-atomicrmw-fsub.ll | 1665 ++----
llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll | 218 +-
.../AMDGPU/float-to-arbitrary-fp-fp8-hw.ll | 325 +-
llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll | 50 +-
llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll | 9 +-
llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll | 50 +-
llvm/test/CodeGen/AMDGPU/fpow.ll | 59 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll | 62 +-
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll | 798 +--
llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll | 54 +-
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 146 +-
llvm/test/CodeGen/AMDGPU/function-returns.ll | 36 +-
.../AMDGPU/gfx-callable-argument-types.ll | 332 +-
.../CodeGen/AMDGPU/global-atomicrmw-fadd.ll | 1663 ++----
.../CodeGen/AMDGPU/global-atomicrmw-fmax.ll | 1671 ++----
.../CodeGen/AMDGPU/global-atomicrmw-fmin.ll | 1671 ++----
.../CodeGen/AMDGPU/global-atomicrmw-fsub.ll | 1667 ++----
.../CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll | 12 +-
.../isel-amdgpu-cs-chain-preserve-cc.ll | 22 +-
.../test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll | 14 +-
llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll | 16 +-
llvm/test/CodeGen/AMDGPU/llvm.log2.ll | 43 +-
llvm/test/CodeGen/AMDGPU/llvm.powi.ll | 17 +-
llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll | 58 +-
llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll | 58 +-
llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll | 64 +-
llvm/test/CodeGen/AMDGPU/usubsat.ll | 28 +-
52 files changed, 15708 insertions(+), 18673 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
index d94b72402dab6..bc94416f09e9f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fptrunc.bf16.ll
@@ -833,3 +833,7 @@ define amdgpu_ps <2 x bfloat> @fptrunc_v2f32_to_v2bf16_v(<2 x float> %a) {
%result = fptrunc <2 x float> %a to <2 x bfloat>
ret <2 x bfloat> %result
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX11: {{.*}}
+; GFX12: {{.*}}
+; GFX1250: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index 6b5a188ab0378..8338865412fdd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -3919,10 +3919,9 @@ define amdgpu_ps half @v_fshr_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
; GFX11-TRUE16-NEXT: s_and_not1_b32 s2, 15, s1
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 15
; GFX11-TRUE16-NEXT: s_and_b32 s0, 0xffff, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, s2, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, s0
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
index ec4239b164dd3..5d850b07311b6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.format.f16.ll
@@ -46,9 +46,8 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__vgpr_voffse
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY4]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
@@ -94,9 +93,8 @@ define amdgpu_ps void @raw_buffer_store_format__sgpr_rsrc__vgpr_val__voffset_409
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY4]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFSET_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFSET_exact [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 4095, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
index c4658df027377..1be7dd9dca0bc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.store.ll
@@ -970,9 +970,8 @@ define amdgpu_ps void @raw_buffer_store__sgpr_rsrc__vgpr_val__vgpr_voffset__sgpr
; GFX12-NEXT: [[COPY4:%[0-9]+]]:vgpr_16 = COPY $vgpr0_lo16
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY4]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: BUFFER_STORE_SHORT_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; GFX12-NEXT: BUFFER_STORE_SHORT_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.buffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
index 051ed6648d01c..8ff542edd3252 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.store.f16.ll
@@ -46,9 +46,8 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret void
@@ -334,8 +333,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr2
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
@@ -345,20 +343,20 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; GFX12-NEXT: [[S_AND_SAVEEXEC_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_SAVEEXEC_B32 killed [[S_AND_B32_]], implicit-def $exec, implicit-def $scc, implicit $exec
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE2]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE1]], [[COPY6]], 0, 94, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -488,8 +486,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr5
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
@@ -499,13 +496,13 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY9]], [[COPY7]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
@@ -515,7 +512,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__vgpr_voffset__vgpr_soff
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -647,9 +644,8 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soff
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr2
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr5
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; GFX12-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_MOV_B32 $exec_lo
; GFX12-NEXT: {{ $}}
; GFX12-NEXT: bb.2:
@@ -659,13 +655,13 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soff
; GFX12-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
; GFX12-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY4]], implicit $exec
- ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
- ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
- ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
- ; GFX12-NEXT: [[COPY10:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub0_sub1
- ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE2]].sub2_sub3
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY10]], [[COPY8]], implicit $exec
- ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; GFX12-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; GFX12-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; GFX12-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; GFX12-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+ ; GFX12-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
; GFX12-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32_xm0_xexec = S_AND_B32 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
; GFX12-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
; GFX12-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_32_xm0_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
@@ -675,7 +671,7 @@ define amdgpu_ps void @raw_tbuffer_store_f16__vgpr_rsrc__sgpr_voffset__vgpr_soff
; GFX12-NEXT: bb.3:
; GFX12-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; GFX12-NEXT: {{ $}}
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY7]], [[REG_SEQUENCE2]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY8]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: $exec_lo = S_XOR_B32_term $exec_lo, [[S_AND_SAVEEXEC_B32_]], implicit-def $scc
; GFX12-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; GFX12-NEXT: {{ $}}
@@ -733,9 +729,8 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 1, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 1)
ret void
@@ -784,9 +779,8 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 2, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 2)
ret void
@@ -835,9 +829,8 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 3, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 3)
ret void
@@ -886,9 +879,8 @@ define amdgpu_ps void @raw_tbuffer_store_f16__sgpr_rsrc__vgpr_voffset__sgpr_soff
; GFX12-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[COPY2]], %subreg.sub1, [[COPY3]], %subreg.sub2, [[COPY4]], %subreg.sub3
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[REG_SEQUENCE1]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: TBUFFER_STORE_FORMAT_D16_X_VBUFFER_OFFEN_exact [[COPY7]], [[COPY5]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 4, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.raw.tbuffer.store.f16(half %val, <4 x i32> %rsrc, i32 %voffset, i32 %soffset, i32 78, i32 4)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
index d23b45c749352..5c9bec56225be 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.store.format.f16.ll
@@ -51,10 +51,9 @@ define amdgpu_ps void @struct_buffer_store_format_f16__vgpr_val__sgpr_rsrc__vgpr
; GFX12-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
; GFX12-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr2
; GFX12-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY $sgpr6
- ; GFX12-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[COPY]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; GFX12-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
- ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
+ ; GFX12-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY]]
+ ; GFX12-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY5]], %subreg.sub0, [[COPY6]], %subreg.sub1
+ ; GFX12-NEXT: BUFFER_STORE_FORMAT_D16_X_VBUFFER_BOTHEN_exact [[COPY8]], [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY7]], 0, 0, 0, implicit $exec :: (dereferenceable store (f16), align 1, addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
call void @llvm.amdgcn.struct.buffer.store.format.f16(half %val, <4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
index af6bfc3fc8559..bf0abb4116062 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-constant.96.ll
@@ -687,21 +687,20 @@ define <12 x i8> @v_load_constant_v12i8_align8(ptr addrspace(4) %ptr) {
; GFX12-UNALIGNED-NEXT: s_wait_samplecnt 0x0
; GFX12-UNALIGNED-NEXT: s_wait_bvhcnt 0x0
; GFX12-UNALIGNED-NEXT: s_wait_kmcnt 0x0
-; GFX12-UNALIGNED-NEXT: global_load_b96 v[0:2], v[0:1], off
+; GFX12-UNALIGNED-NEXT: global_load_b96 v[12:14], v[0:1], off
; GFX12-UNALIGNED-NEXT: s_wait_loadcnt 0x0
-; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v13, 8, v0
-; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v12, 16, v0
-; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v1.l, v13.l
-; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v2.l, v12.l
+; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v1, 8, v12
+; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v3, 24, v12
+; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v5, 8, v13
+; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v7, 24, v13
+; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v9, 8, v14
+; GFX12-UNALIGNED-NEXT: v_lshrrev_b32_e32 v11, 24, v14
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v0.l, v12.l
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v2.l, v12.h
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v4.l, v13.l
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v6.l, v13.h
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v8.l, v14.l
+; GFX12-UNALIGNED-NEXT: v_mov_b16_e32 v10.l, v14.h
; GFX12-UNALIGNED-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-NOUNALIGNED-LABEL: v_load_constant_v12i8_align8:
@@ -711,39 +710,38 @@ define <12 x i8> @v_load_constant_v12i8_align8(ptr addrspace(4) %ptr) {
; GFX12-NOUNALIGNED-NEXT: s_wait_samplecnt 0x0
; GFX12-NOUNALIGNED-NEXT: s_wait_bvhcnt 0x0
; GFX12-NOUNALIGNED-NEXT: s_wait_kmcnt 0x0
-; GFX12-NOUNALIGNED-NEXT: global_load_b96 v[0:2], v[0:1], off
+; GFX12-NOUNALIGNED-NEXT: global_load_b96 v[12:14], v[0:1], off
; GFX12-NOUNALIGNED-NEXT: s_wait_loadcnt 0x0
-; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v13, 8, v0
-; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v12, 16, v0
-; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v5, 8, v1
-; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v7, 24, v1
-; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v8.l, v2.l
-; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v1.l, v13.l
-; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v2.l, v12.l
+; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v1, 8, v12
+; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v3, 24, v12
+; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v5, 8, v13
+; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v7, 24, v13
+; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v9, 8, v14
+; GFX12-NOUNALIGNED-NEXT: v_lshrrev_b32_e32 v11, 24, v14
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v0.l, v12.l
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v2.l, v12.h
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v4.l, v13.l
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v6.l, v13.h
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v8.l, v14.l
+; GFX12-NOUNALIGNED-NEXT: v_mov_b16_e32 v10.l, v14.h
; GFX12-NOUNALIGNED-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-LABEL: v_load_constant_v12i8_align8:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b96 v[0:2], v[0:1], off nv
+; GFX1250-NEXT: global_load_b96 v[12:14], v[0:1], off nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_dual_lshrrev_b32 v13, 8, v0 :: v_dual_lshrrev_b32 v12, 16, v0
-; GFX1250-NEXT: v_dual_lshrrev_b32 v3, 24, v0 :: v_dual_lshrrev_b32 v5, 8, v1
-; GFX1250-NEXT: v_dual_lshrrev_b32 v6, 16, v1 :: v_dual_lshrrev_b32 v7, 24, v1
-; GFX1250-NEXT: v_dual_lshrrev_b32 v9, 8, v2 :: v_dual_lshrrev_b32 v10, 16, v2
-; GFX1250-NEXT: v_lshrrev_b32_e32 v11, 24, v2
-; GFX1250-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX1250-NEXT: v_mov_b16_e32 v8.l, v2.l
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_mov_b16_e32 v1.l, v13.l
-; GFX1250-NEXT: v_mov_b16_e32 v2.l, v12.l
+; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 8, v12 :: v_dual_lshrrev_b32 v3, 24, v12
+; GFX1250-NEXT: v_dual_lshrrev_b32 v5, 8, v13 :: v_dual_lshrrev_b32 v7, 24, v13
+; GFX1250-NEXT: v_dual_lshrrev_b32 v9, 8, v14 :: v_dual_lshrrev_b32 v11, 24, v14
+; GFX1250-NEXT: v_mov_b16_e32 v0.l, v12.l
+; GFX1250-NEXT: v_mov_b16_e32 v2.l, v12.h
+; GFX1250-NEXT: v_mov_b16_e32 v4.l, v13.l
+; GFX1250-NEXT: v_mov_b16_e32 v6.l, v13.h
+; GFX1250-NEXT: v_mov_b16_e32 v8.l, v14.l
+; GFX1250-NEXT: v_mov_b16_e32 v10.l, v14.h
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
; GFX9-LABEL: v_load_constant_v12i8_align8:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index 991071ac55bc5..0e097325025d0 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -2936,85 +2936,79 @@ define <4 x i32> @bitcast_v8bf16_to_v4i32(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v12 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v2, v7, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
; GFX11-TRUE16-NEXT: .LBB22_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -4168,76 +4162,85 @@ define inreg <16 x i8> @bitcast_v4i32_to_v16i8_scalar(<4 x i32> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s18, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB25_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s8 :: v_dual_mov_b32 v0, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s6
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s6
; GFX11-TRUE16-NEXT: s_branch .LBB25_3
; GFX11-TRUE16-NEXT: .LBB25_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s18, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: .LBB25_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s18, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB25_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
; GFX11-TRUE16-NEXT: s_add_i32 s3, s3, 3
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s4 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s3, 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s6 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v15, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v13, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s5 :: v_dual_mov_b32 v9, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v7, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: .LBB25_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s15
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, s10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, s9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, s8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i32_to_v16i8_scalar:
@@ -5135,84 +5138,168 @@ define inreg <4 x i32> @bitcast_v16i8_to_v4i32_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB27_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v16i8_to_v4i32_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_branch .LBB27_3
-; GFX11-NEXT: .LBB27_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB27_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB27_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: s_add_i32 s24, s24, 3
-; GFX11-NEXT: s_add_i32 s26, s26, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB27_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v4i32_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB27_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
+; GFX11-TRUE16-NEXT: s_branch .LBB27_3
+; GFX11-TRUE16-NEXT: .LBB27_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB27_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: .LBB27_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v4i32_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB27_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-FAKE16-NEXT: s_branch .LBB27_3
+; GFX11-FAKE16-NEXT: .LBB27_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB27_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-FAKE16-NEXT: .LBB27_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7773,85 +7860,79 @@ define <4 x float> @bitcast_v8bf16_to_v4f32(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v12 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v2, v7, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
; GFX11-TRUE16-NEXT: .LBB46_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -9021,79 +9102,81 @@ define inreg <16 x i8> @bitcast_v4f32_to_v16i8_scalar(<4 x float> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v16, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v18, s0
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
; GFX11-TRUE16-NEXT: s_branch .LBB49_3
; GFX11-TRUE16-NEXT: .LBB49_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, s3, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, s2, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, s0, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v19, s1, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v17, s3, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v16, s2, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v18, s0, 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB49_6
-; GFX11-TRUE16-NEXT: .LBB49_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v7, s14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v15, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v13, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s10 :: v_dual_mov_b32 v5, s4
-; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f32_to_v16i8_scalar:
@@ -10000,107 +10083,191 @@ define inreg <4 x float> @bitcast_v16i8_to_v4f32_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v16i8_to_v4f32_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_branch .LBB51_3
-; GFX11-NEXT: .LBB51_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB51_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB51_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: s_add_i32 s24, s24, 3
-; GFX11-NEXT: s_add_i32 s26, s26, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB51_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
- %cmp = icmp eq i32 %b, 0
- br i1 %cmp, label %cmp.true, label %cmp.false
-
-cmp.true:
- %a1 = add <16 x i8> %a, splat (i8 3)
- %a2 = bitcast <16 x i8> %a1 to <4 x float>
- br label %end
-
-cmp.false:
- %a3 = bitcast <16 x i8> %a to <4 x float>
- br label %end
-
-end:
- %phi = phi <4 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
- ret <4 x float> %phi
-}
-
-define <2 x double> @bitcast_v2i64_to_v2f64(<2 x i64> %a, i32 %b) #0 {
-; SI-LABEL: bitcast_v2i64_to_v2f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
-; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v4f32_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
+; GFX11-TRUE16-NEXT: s_branch .LBB51_3
+; GFX11-TRUE16-NEXT: .LBB51_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB51_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: .LBB51_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v4f32_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-FAKE16-NEXT: s_branch .LBB51_3
+; GFX11-FAKE16-NEXT: .LBB51_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB51_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-FAKE16-NEXT: .LBB51_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %cmp = icmp eq i32 %b, 0
+ br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+ %a1 = add <16 x i8> %a, splat (i8 3)
+ %a2 = bitcast <16 x i8> %a1 to <4 x float>
+ br label %end
+
+cmp.false:
+ %a3 = bitcast <16 x i8> %a to <4 x float>
+ br label %end
+
+end:
+ %phi = phi <4 x float> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+ ret <4 x float> %phi
+}
+
+define <2 x double> @bitcast_v2i64_to_v2f64(<2 x i64> %a, i32 %b) #0 {
+; SI-LABEL: bitcast_v2i64_to_v2f64:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4
+; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; SI-NEXT: s_andn2_saveexec_b64 s[4:5], s[4:5]
; SI-NEXT: ; %bb.1: ; %cmp.true
@@ -12206,85 +12373,79 @@ define <2 x i64> @bitcast_v8bf16_to_v2i64(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v12 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v2, v7, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
; GFX11-TRUE16-NEXT: .LBB66_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -13438,76 +13599,87 @@ define inreg <16 x i8> @bitcast_v2i64_to_v16i8_scalar(<2 x i64> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s18, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB69_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s8 :: v_dual_mov_b32 v0, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v12, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s6
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s6
; GFX11-TRUE16-NEXT: s_branch .LBB69_3
; GFX11-TRUE16-NEXT: .LBB69_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s18, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: .LBB69_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s18, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB69_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, 0
; GFX11-TRUE16-NEXT: s_add_u32 s2, s2, 3
; GFX11-TRUE16-NEXT: s_addc_u32 s3, s3, 0
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v15, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v14, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v13, s6
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v9, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v5, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s2 :: v_dual_mov_b32 v6, s3
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX11-TRUE16-NEXT: .LBB69_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s15
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, s10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, s9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, s8
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2i64_to_v16i8_scalar:
@@ -14405,84 +14577,168 @@ define inreg <2 x i64> @bitcast_v16i8_to_v2i64_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v16i8_to_v2i64_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_branch .LBB71_3
-; GFX11-NEXT: .LBB71_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB71_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: s_add_i32 s24, s24, 3
-; GFX11-NEXT: s_add_i32 s26, s26, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB71_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v2i64_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
+; GFX11-TRUE16-NEXT: s_branch .LBB71_3
+; GFX11-TRUE16-NEXT: .LBB71_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB71_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: .LBB71_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v2i64_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-FAKE16-NEXT: s_branch .LBB71_3
+; GFX11-FAKE16-NEXT: .LBB71_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB71_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-FAKE16-NEXT: .LBB71_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -16195,85 +16451,79 @@ define <2 x double> @bitcast_v8bf16_to_v2f64(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v7, v8 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v8, v12 :: v_dual_add_f32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v9.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v4, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v2, v7, v8
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v7, 0x7fff
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v6.l
; GFX11-TRUE16-NEXT: .LBB82_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -17433,80 +17683,82 @@ define inreg <16 x i8> @bitcast_v2f64_to_v16i8_scalar(<2 x double> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB85_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v16, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v18, s0
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
; GFX11-TRUE16-NEXT: s_branch .LBB85_3
; GFX11-TRUE16-NEXT: .LBB85_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: .LBB85_3: ; %Flow
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB85_5
-; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f64 v[11:12], s[2:3], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], s[0:1], 1.0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB85_6
-; GFX11-TRUE16-NEXT: .LBB85_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v12, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s18 :: v_dual_mov_b32 v1, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v9, s15
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_mov_b32 v14, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s6 :: v_dual_mov_b32 v16, s12
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s14 :: v_dual_mov_b32 v6, s10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s11 :: v_dual_mov_b32 v8, s9
-; GFX11-TRUE16-NEXT: .LBB85_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: bitcast_v2f64_to_v16i8_scalar:
-; GFX11-FAKE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: .LBB85_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB85_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f64 v[16:17], s[2:3], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], s[0:1], 1.0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-TRUE16-NEXT: .LBB85_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v2f64_to_v16i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
; GFX11-FAKE16-NEXT: s_mov_b32 s8, 0
@@ -18406,84 +18658,168 @@ define inreg <2 x double> @bitcast_v16i8_to_v2f64_scalar(<16 x i8> inreg %a, i32
; GFX9-NEXT: .LBB87_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v16i8_to_v2f64_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB87_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_branch .LBB87_3
-; GFX11-NEXT: .LBB87_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB87_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB87_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: s_add_i32 s24, s24, 3
-; GFX11-NEXT: s_add_i32 s26, s26, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB87_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v2f64_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB87_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
+; GFX11-TRUE16-NEXT: s_branch .LBB87_3
+; GFX11-TRUE16-NEXT: .LBB87_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB87_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: .LBB87_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v2f64_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB87_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-FAKE16-NEXT: s_branch .LBB87_3
+; GFX11-FAKE16-NEXT: .LBB87_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB87_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-FAKE16-NEXT: .LBB87_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -19804,89 +20140,82 @@ define <8 x i16> @bitcast_v8bf16_to_v8i16(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v12, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v5, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v6, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v11, v12 :: v_dual_add_f32 v0, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v3, 16, 1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v5, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v8, v13 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v13, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v9, v13, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v5.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v12, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v7.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v12, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v10, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v6.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v3.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v10, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v4.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v7.l
; GFX11-TRUE16-NEXT: .LBB94_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -21151,79 +21480,81 @@ define inreg <16 x i8> @bitcast_v8i16_to_v16i8_scalar(<8 x i16> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v16, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v18, s0
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
; GFX11-TRUE16-NEXT: s_branch .LBB97_3
; GFX11-TRUE16-NEXT: .LBB97_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: .LBB97_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v12, s3, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, s2, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v19, s1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v17, s3, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v16, s2, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v18, s0, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB97_6
-; GFX11-TRUE16-NEXT: .LBB97_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v7, s14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v15, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v13, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s10 :: v_dual_mov_b32 v5, s4
-; GFX11-TRUE16-NEXT: .LBB97_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-TRUE16-NEXT: .LBB97_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8i16_to_v16i8_scalar:
@@ -22178,84 +22509,168 @@ define inreg <8 x i16> @bitcast_v16i8_to_v8i16_scalar(<16 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v16i8_to_v8i16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB99_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_branch .LBB99_3
-; GFX11-NEXT: .LBB99_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB99_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB99_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: s_add_i32 s24, s24, 3
-; GFX11-NEXT: s_add_i32 s26, s26, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB99_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8i16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
+; GFX11-TRUE16-NEXT: s_branch .LBB99_3
+; GFX11-TRUE16-NEXT: .LBB99_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB99_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: .LBB99_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v8i16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-FAKE16-NEXT: s_branch .LBB99_3
+; GFX11-FAKE16-NEXT: .LBB99_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB99_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-FAKE16-NEXT: .LBB99_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -22926,87 +23341,79 @@ define <8 x half> @bitcast_v8bf16_to_v8f16(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v7, v8 :: v_dual_and_b32 v5, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v7 :: v_dual_add_f32 v6, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v7, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v10, v12 :: v_dual_and_b32 v6, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v7, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v10, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v11, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v11, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v9, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v13, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v7.l
; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
@@ -24320,79 +24727,81 @@ define inreg <16 x i8> @bitcast_v8f16_to_v16i8_scalar(<8 x half> inreg %a, i32 i
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB105_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v16, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v18, s0
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
; GFX11-TRUE16-NEXT: s_branch .LBB105_3
; GFX11-TRUE16-NEXT: .LBB105_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: .LBB105_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v12, 0x200, s3 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, s2 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v19, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v17, 0x200, s3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v16, 0x200, s2 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v18, 0x200, s0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[13:14], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB105_6
-; GFX11-TRUE16-NEXT: .LBB105_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s2 :: v_dual_mov_b32 v12, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s18 :: v_dual_mov_b32 v2, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v7, s14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s11 :: v_dual_mov_b32 v15, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v13, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s10 :: v_dual_mov_b32 v5, s4
-; GFX11-TRUE16-NEXT: .LBB105_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-TRUE16-NEXT: .LBB105_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8f16_to_v16i8_scalar:
@@ -25347,84 +25756,168 @@ define inreg <8 x half> @bitcast_v16i8_to_v8f16_scalar(<16 x i8> inreg %a, i32 i
; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v16i8_to_v8f16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB107_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_branch .LBB107_3
-; GFX11-NEXT: .LBB107_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB107_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB107_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: s_add_i32 s24, s24, 3
-; GFX11-NEXT: s_add_i32 s26, s26, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB107_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8f16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
+; GFX11-TRUE16-NEXT: s_branch .LBB107_3
+; GFX11-TRUE16-NEXT: .LBB107_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB107_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: .LBB107_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v8f16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-FAKE16-NEXT: s_branch .LBB107_3
+; GFX11-FAKE16-NEXT: .LBB107_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB107_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-FAKE16-NEXT: .LBB107_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -25852,94 +26345,87 @@ define <16 x i8> @bitcast_v8bf16_to_v16i8(<8 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v19
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v18.l
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v18
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v19
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v0, v5, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v9, v7 :: v_dual_and_b32 v9, 0xffff0000, v16
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v8, v10 :: v_dual_lshlrev_b32 v1, 16, v17
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v10, 16, v16
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v7 :: v_dual_add_f32 v9, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v5, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v2.l
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v8, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v14.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v10.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
@@ -26515,154 +27001,153 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s3, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s17, s3, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s3, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s18, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s15, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s16, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, s5 :: v_dual_mov_b32 v14, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v2, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v16, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v17, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v18, s0
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
; GFX11-TRUE16-NEXT: s_branch .LBB109_3
; GFX11-TRUE16-NEXT: .LBB109_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr16
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr15
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr18
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr17
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
+; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: .LBB109_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB109_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s1
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s4
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s4, 0, s3
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s4
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s4
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s3
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v10, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s3, 16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, v11, v10
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x7fff, v11
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v2, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v5, v7
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v9, v7
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v6.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v5.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s2, 16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v2.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v10, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v1, v9 :: v_dual_add_nc_u32 v10, v12, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v10
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v9
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v9, v11
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v14.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v11, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[17:18]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v8.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v13.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v10.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v18
-; GFX11-TRUE16-NEXT: s_branch .LBB109_6
-; GFX11-TRUE16-NEXT: .LBB109_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v13, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s18 :: v_dual_mov_b32 v5, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v9, s14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v15, s11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s16 :: v_dual_mov_b32 v1, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v7, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s12 :: v_dual_mov_b32 v11, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v3, s4
-; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v14.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v10.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[16:17]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v16
+; GFX11-TRUE16-NEXT: .LBB109_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v17.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8bf16_to_v16i8_scalar:
@@ -27689,84 +28174,168 @@ define inreg <8 x bfloat> @bitcast_v16i8_to_v8bf16_scalar(<16 x i8> inreg %a, i3
; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v16i8_to_v8bf16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s28, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB111_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s26, s27, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v8, s24, s25, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v4
-; GFX11-NEXT: v_or_b32_e32 v3, v8, v6
-; GFX11-NEXT: s_branch .LBB111_3
-; GFX11-NEXT: .LBB111_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB111_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB111_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: s_add_i32 s24, s24, 3
-; GFX11-NEXT: s_add_i32 s26, s26, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v6, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v7, s24, s25, v0
-; GFX11-NEXT: v_perm_b32 v0, s26, s27, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
-; GFX11-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-NEXT: v_lshlrev_b32_e32 v8, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT: v_or_b32_e32 v3, v7, v8
-; GFX11-NEXT: .LBB111_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v16i8_to_v8bf16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s27 :: v_dual_mov_b32 v7, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v10, s24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s23 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s21 :: v_dual_mov_b32 v9, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s20 :: v_dual_mov_b32 v13, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s17 :: v_dual_mov_b32 v15, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s16 :: v_dual_mov_b32 v17, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s2 :: v_dual_mov_b32 v19, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v18, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v13, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v16, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_perm_b32 v22, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v20, 16, v20
+; GFX11-TRUE16-NEXT: v_perm_b32 v23, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v21, 16, v21
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v22, v20
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v23, v21
+; GFX11-TRUE16-NEXT: s_branch .LBB111_3
+; GFX11-TRUE16-NEXT: .LBB111_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB111_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v19
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v12, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v11, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v7, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, v5, v4
+; GFX11-TRUE16-NEXT: .LBB111_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v16i8_to_v8bf16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s28, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v8, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v8, v6
+; GFX11-FAKE16-NEXT: s_branch .LBB111_3
+; GFX11-FAKE16-NEXT: .LBB111_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB111_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s24, s24, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s26, s26, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v7, s24, s25, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s26, s27, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v6
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v7
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v7, v8
+; GFX11-FAKE16-NEXT: .LBB111_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
index e34ffdf0fdff9..a3f05ace62565 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.16bit.ll
@@ -161,20 +161,24 @@ define inreg half @bitcast_i16_to_f16_scalar(i16 inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_2
-; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB1_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_branch .LBB1_3
+; GFX11-TRUE16-NEXT: .LBB1_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: .LBB1_2: ; %Flow
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: .LBB1_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-TRUE16-NEXT: .LBB1_4: ; %end
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB1_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: .LBB1_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i16_to_f16_scalar:
@@ -390,20 +394,22 @@ define inreg i16 @bitcast_f16_to_i16_scalar(half inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB3_2
-; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB3_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_branch .LBB3_3
+; GFX11-TRUE16-NEXT: .LBB3_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: .LBB3_2: ; %Flow
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: .LBB3_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB3_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB3_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, 0x200, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB3_4:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: .LBB3_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_i16_scalar:
@@ -605,20 +611,24 @@ define inreg bfloat @bitcast_i16_to_bf16_scalar(i16 inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_2
-; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB5_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_branch .LBB5_3
+; GFX11-TRUE16-NEXT: .LBB5_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: .LBB5_2: ; %Flow
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: .LBB5_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-TRUE16-NEXT: .LBB5_4: ; %end
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v0
+; GFX11-TRUE16-NEXT: .LBB5_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i16_to_bf16_scalar:
@@ -864,16 +874,20 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_2
-; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB7_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_branch .LBB7_3
+; GFX11-TRUE16-NEXT: .LBB7_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: .LBB7_2: ; %Flow
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: .LBB7_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -886,9 +900,7 @@ define inreg i16 @bitcast_bf16_to_i16_scalar(bfloat inreg %a, i32 inreg %b) #0 {
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB7_4:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: .LBB7_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_bf16_to_i16_scalar:
@@ -1116,20 +1128,22 @@ define inreg bfloat @bitcast_f16_to_bf16_scalar(half inreg %a, i32 inreg %b) #0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB9_2
-; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB9_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_branch .LBB9_3
+; GFX11-TRUE16-NEXT: .LBB9_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: .LBB9_2: ; %Flow
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: .LBB9_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB9_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB9_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, 0x200, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB9_4:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: .LBB9_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f16_to_bf16_scalar:
@@ -1375,16 +1389,20 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_2
-; GFX11-TRUE16-NEXT: ; %bb.1:
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB11_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_branch .LBB11_3
+; GFX11-TRUE16-NEXT: .LBB11_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
-; GFX11-TRUE16-NEXT: .LBB11_2: ; %Flow
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: .LBB11_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
-; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB11_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s0
@@ -1397,9 +1415,7 @@ define inreg half @bitcast_bf16_to_f16_scalar(bfloat inreg %a, i32 inreg %b) #0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB11_4:
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX11-TRUE16-NEXT: .LBB11_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_bf16_to_f16_scalar:
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 326df5d06e29f..353912ea0d99b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -4031,159 +4031,162 @@ define <8 x i32> @bitcast_v16bf16_to_v8i32(<16 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v8i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v14 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v15, v12, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v11, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.h
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v6, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v11, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v15, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v16, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v12, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v12, v6, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v13, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v11, v13 :: v_dual_and_b32 v13, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v4, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v10
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v9, v14 :: v_dual_add_f32 v12, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v12, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v11, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v9.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_add_f32 v10, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v12, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.h
-; GFX11-TRUE16-NEXT: .LBB22_4: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v13, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v10.l
+; GFX11-TRUE16-NEXT: .LBB22_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v12 :: v_dual_mov_b32 v5, v13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v14 :: v_dual_mov_b32 v7, v15
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v8i32:
@@ -5549,95 +5552,109 @@ define <32 x i8> @bitcast_v8i32_to_v32i8(<8 x i32> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v5 :: v_dual_mov_b32 v26, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB24_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB24_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v39, 3, v39
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v37, 3, v37
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v35, 3, v35
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v33, 3, v33
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v32, 3, v32
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 3, v27
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v26, 3, v26
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v19
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 3, v18
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v34, 3, v34
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v36, 3, v36
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v38, 3, v38
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB24_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v19.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v27.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v32.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v33.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8i32_to_v32i8:
@@ -7006,22 +7023,22 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v8i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB26_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -7032,11 +7049,11 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB26_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v37, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v35, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
@@ -7051,21 +7068,21 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v12, 16, v11
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
@@ -7088,26 +7105,26 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
; GFX11-TRUE16-NEXT: .LBB26_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v38.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v36.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v34.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v10.l, 3
@@ -7157,7 +7174,7 @@ define <8 x i32> @bitcast_v32i8_to_v8i32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v9.l, v7.l
@@ -11816,159 +11833,162 @@ define <8 x float> @bitcast_v16bf16_to_v8f32(<16 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v8f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v14 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v15, v12, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v11, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.h
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v6, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v11, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v15, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v16, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v12, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v12, v6, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v13, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v11, v13 :: v_dual_and_b32 v13, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v4, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v10
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v9, v14 :: v_dual_add_f32 v12, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v12, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v11, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v9.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_add_f32 v10, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v12, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.h
-; GFX11-TRUE16-NEXT: .LBB46_4: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v13, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v10.l
+; GFX11-TRUE16-NEXT: .LBB46_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v12 :: v_dual_mov_b32 v5, v13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v14 :: v_dual_mov_b32 v7, v15
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v8f32:
@@ -13334,93 +13354,107 @@ define <32 x i8> @bitcast_v8f32_to_v32i8(<8 x float> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v5 :: v_dual_mov_b32 v26, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB48_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v11, 1.0, v11 :: v_dual_add_f32 v32, 1.0, v32
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 1.0, v33 :: v_dual_add_f32 v10, 1.0, v10
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v19, 1.0, v19 :: v_dual_add_f32 v26, 1.0, v26
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v27, 1.0, v27 :: v_dual_add_f32 v18, 1.0, v18
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v39, 1.0, v39 :: v_dual_add_f32 v32, 1.0, v32
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v37, 1.0, v37 :: v_dual_add_f32 v34, 1.0, v34
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v35, 1.0, v35 :: v_dual_add_f32 v36, 1.0, v36
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v33, 1.0, v33 :: v_dual_add_f32 v38, 1.0, v38
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB48_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v19.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v27.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v32.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v33.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v8f32_to_v32i8:
@@ -14011,46 +14045,47 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v34, s16
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v24, s8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v8, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v32, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v36, s2
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v32, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v38, s0
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s6 :: v_dual_mov_b32 v20, s17
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s6 :: v_dual_mov_b32 v33, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v19, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s0
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v33, s19
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s17
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v39, s1
; GFX11-TRUE16-NEXT: s_branch .LBB49_3
; GFX11-TRUE16-NEXT: .LBB49_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
@@ -14068,45 +14103,45 @@ define inreg <32 x i8> @bitcast_v8f32_to_v32i8_scalar(<8 x float> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, s3, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v20, s17, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v39, s1, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v37, s3, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v35, s17, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v33, s19, 1.0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v32, s18, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v19, s16, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, s2, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, s0, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v34, s16, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v36, s2, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v38, s0, 1.0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -14821,22 +14856,22 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v8f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -14847,11 +14882,11 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v37, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v35, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
@@ -14866,21 +14901,21 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v12, 16, v11
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
@@ -14903,26 +14938,26 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v38.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v36.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v34.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v10.l, 3
@@ -14972,7 +15007,7 @@ define <8 x float> @bitcast_v32i8_to_v8f32(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v9.l, v7.l
@@ -19087,159 +19122,162 @@ define <4 x i64> @bitcast_v16bf16_to_v4i64(<16 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v4i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v14 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v15, v12, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v11, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.h
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v6, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v11, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v15, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v16, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v12, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v12, v6, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v13, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v11, v13 :: v_dual_and_b32 v13, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v4, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v10
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v9, v14 :: v_dual_add_f32 v12, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v12, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v11, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v9.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_add_f32 v10, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v12, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.h
-; GFX11-TRUE16-NEXT: .LBB66_4: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v13, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v10.l
+; GFX11-TRUE16-NEXT: .LBB66_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v12 :: v_dual_mov_b32 v5, v13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v14 :: v_dual_mov_b32 v7, v15
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v4i64:
@@ -20605,98 +20643,112 @@ define <32 x i8> @bitcast_v4i64_to_v32i8(<4 x i64> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v5 :: v_dual_mov_b32 v26, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB68_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB68_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_co_u32 v18, vcc_lo, v18, 3
+; GFX11-TRUE16-NEXT: v_add_co_u32 v36, vcc_lo, v36, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v19, null, 0, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_co_u32 v26, vcc_lo, v26, 3
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v27, null, 0, v27, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v37, null, 0, v37, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_u32 v34, vcc_lo, v34, 3
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v35, null, 0, v35, vcc_lo
; GFX11-TRUE16-NEXT: v_add_co_u32 v32, vcc_lo, v32, 3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v33, null, 0, v33, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_co_u32 v10, vcc_lo, v10, 3
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v11, null, 0, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_co_u32 v38, vcc_lo, v38, 3
+; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v39, null, 0, v39, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB68_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v19.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v27.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v32.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v33.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i64_to_v32i8:
@@ -22073,22 +22125,22 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v4i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB70_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -22099,11 +22151,11 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB70_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v37, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v35, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
@@ -22118,21 +22170,21 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v12, 16, v11
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
@@ -22155,26 +22207,26 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB70_2
; GFX11-TRUE16-NEXT: .LBB70_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v38.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v36.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v34.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v10.l, 3
@@ -22224,7 +22276,7 @@ define <4 x i64> @bitcast_v32i8_to_v4i64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v9.l, v7.l
@@ -25813,159 +25865,162 @@ define <4 x double> @bitcast_v16bf16_to_v4f64(<16 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v4f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v15, v7 :: v_dual_mov_b32 v14, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v5 :: v_dual_mov_b32 v12, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v3 :: v_dual_mov_b32 v10, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v1 :: v_dual_mov_b32 v8, v0
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_4
-; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
+; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v14 :: v_dual_and_b32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v6, 16, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v15, v12, v13
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v8.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v6, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v14, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v11, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v6.h
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v6, 0x40c00000, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v6, v11, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v5.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v9, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v12, v14, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v15, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v16, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v12, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_lshlrev_b32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_cndmask_b32 v12, v6, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v11
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v4.h
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v6, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v13, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v10, v11, v13 :: v_dual_and_b32 v13, 0xffff0000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v4, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v10, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v3.h
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v2.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v10
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v9, v14 :: v_dual_add_f32 v12, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v12, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v12, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_lshlrev_b32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_add_f32 v11, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v10, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v9.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v9, v11 :: v_dual_and_b32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v12, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v9, v9, v11 :: v_dual_add_f32 v10, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v9
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v13, v2, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v9, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v12, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v10, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v2, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.h
-; GFX11-TRUE16-NEXT: .LBB82_4: ; %end
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v13, v16, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v10.l
+; GFX11-TRUE16-NEXT: .LBB82_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v12 :: v_dual_mov_b32 v5, v13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v14 :: v_dual_mov_b32 v7, v15
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v4f64:
@@ -27325,93 +27380,107 @@ define <32 x i8> @bitcast_v4f64_to_v32i8(<4 x double> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v5 :: v_dual_mov_b32 v26, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB84_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB84_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f64 v[32:33], v[32:33], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[26:27], v[26:27], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[18:19], v[18:19], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[10:11], v[10:11], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[34:35], v[34:35], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[36:37], v[36:37], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[38:39], v[38:39], 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB84_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v19.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v27.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v32.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v33.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f64_to_v32i8:
@@ -27990,54 +28059,54 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v34, s16
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s8 :: v_dual_mov_b32 v12, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v28, s19
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v32, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v36, s2
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v20, s17
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v32, s6
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v38, s0
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v34, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v35, s8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v27, s18
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v33, s19
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s17
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v39, s1
; GFX11-TRUE16-NEXT: s_branch .LBB85_3
; GFX11-TRUE16-NEXT: .LBB85_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
@@ -28048,45 +28117,45 @@ define inreg <32 x i8> @bitcast_v4f64_to_v32i8_scalar(<4 x double> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB85_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f64 v[27:28], s[18:19], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[19:20], s[16:17], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[11:12], s[2:3], 1.0
-; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], s[0:1], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[32:33], s[18:19], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[34:35], s[16:17], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[36:37], s[2:3], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[38:39], s[0:1], 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[33:34], 24, v[19:20]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB85_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f64_to_v32i8_scalar:
@@ -28797,22 +28866,22 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v4f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v48, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v48
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB86_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -28823,11 +28892,11 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB86_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v39, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v37, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v35, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v8, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
@@ -28842,21 +28911,21 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v11, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v30, v39, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v4, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v12, 16, v11
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
@@ -28879,26 +28948,26 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB86_2
; GFX11-TRUE16-NEXT: .LBB86_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v39.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v38.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v37.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v35.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v36.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v34.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v33.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v8.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v8.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, v10.l, 3
@@ -28948,7 +29017,7 @@ define <4 x double> @bitcast_v32i8_to_v4f64(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v6.h, v6.l
; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v9.l, v7.l
@@ -32109,139 +32178,151 @@ define <16 x i16> @bitcast_v16bf16_to_v16i16(<16 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v8, 16, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v8, v11, v12 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v10, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v12, 0x40c00000, v12 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v11, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v13, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v14, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v14, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v16, 16, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v11, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
; GFX11-TRUE16-NEXT: v_add_f32_e32 v10, 0x40c00000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v10, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v11, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v14, v10, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v11, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v14, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v15, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v11, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v14, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v12, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v14, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v11, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v14, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v5
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v11, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v11, 0x40c00000, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v12.h
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v18, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v11, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v1, v12, v16
; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v5, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v14, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v9.h
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v17, v11, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v14, v15 :: v_dual_lshlrev_b32 v17, 16, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v19, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v13.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v16, v18 :: v_dual_lshlrev_b32 v18, 16, v7
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v17
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v14, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v15, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v14, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v15
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v15 :: v_dual_lshlrev_b32 v15, 16, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v18, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v12, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v17, v14, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v11.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v16, v17, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v12, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_add_f32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v15
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v17, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v20, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v23, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v12, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v20, v20, v7, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v17, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v14
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v18, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v15, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v15, v18, v19, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v16.l
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v15, v21, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v17, v20, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v11.l
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v6, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v12, v22, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v14.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v16, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v15.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v20, v23, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v7.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v17, v19, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v18.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v13.l
; GFX11-TRUE16-NEXT: .LBB94_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -33707,95 +33788,109 @@ define <32 x i8> @bitcast_v16i16_to_v32i8(<16 x i16> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v5 :: v_dual_mov_b32 v26, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB96_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB96_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v39, v39, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v37, v37, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v35, v35, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v33, v33, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v32, v32, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v27, v27, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v26, v26, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, v11, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v19, v19, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v18, v18, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v10, v10, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v34, v34, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v36, v36, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v38, v38, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB96_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v19.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v27.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v32.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v33.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16i16_to_v32i8:
@@ -34456,46 +34551,47 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v34, s16
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v24, s8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v8, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v32, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v36, s2
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v32, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v38, s0
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s6 :: v_dual_mov_b32 v20, s17
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s6 :: v_dual_mov_b32 v33, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v19, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s0
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v33, s19
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s17
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v39, s1
; GFX11-TRUE16-NEXT: s_branch .LBB97_3
; GFX11-TRUE16-NEXT: .LBB97_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
@@ -34513,45 +34609,45 @@ define inreg <32 x i8> @bitcast_v16i16_to_v32i8_scalar(<16 x i16> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v12, s3, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v20, s17, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v39, s1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v37, s3, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v35, s17, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v33, s19, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v32, s18, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v19, s16, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v11, s2, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v34, s16, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v36, s2, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v38, s0, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB97_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -35313,44 +35409,29 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v16i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: scratch_load_b32 v1, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v23.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v27.h
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v31.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v37
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB98_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -35360,133 +35441,133 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB98_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v20, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v28, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v31, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v37, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v14, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v31, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v12, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v12, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v34, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v8.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v8, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v26.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v10.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v34, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v13.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v9.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v36, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v12.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v11.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB98_2
; GFX11-TRUE16-NEXT: .LBB98_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v10.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v13.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v22.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v10.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v12.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v15.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.h, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v31.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v32.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v12.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v10.h, v1.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v10.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v15.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v20.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v31.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v11.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v14.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v8.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.l, v1.h
@@ -36316,63 +36397,65 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v11
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v10
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, s4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v19, s6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v31, s7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s4 :: v_dual_mov_b32 v31, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v22, s6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v34, s7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s8 :: v_dual_mov_b32 v25, s4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s5 :: v_dual_mov_b32 v27, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v25, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s6 :: v_dual_mov_b32 v38, s7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s8
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s17 :: v_dual_mov_b32 v12, s3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v14
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v38, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, s4 :: v_dual_mov_b32 v30, s8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v36, s5 :: v_dual_mov_b32 v39, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v35, s28
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s8 :: v_dual_mov_b32 v21, s27
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, s29 :: v_dual_mov_b32 v29, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s25 :: v_dual_mov_b32 v8, s23
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s24 :: v_dual_mov_b32 v24, s21
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s22 :: v_dual_mov_b32 v30, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s27 :: v_dual_mov_b32 v19, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v21, s29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s24 :: v_dual_mov_b32 v33, s28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s23 :: v_dual_mov_b32 v23, s25
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s21 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v28, s20
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s19 :: v_dual_mov_b32 v15, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s16 :: v_dual_mov_b32 v18, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s1 :: v_dual_mov_b32 v17, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s16 :: v_dual_mov_b32 v14, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s2 :: v_dual_mov_b32 v16, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v18, v12, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v17, v12, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v15, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v11, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v29, v21, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v20, v32, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v26, v38, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v48, v25, v34, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v49, v16, v28, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v17, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v26, v20, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v19, v30, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v27, v37, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v48, v25, v35, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v49, v18, v31, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v16, v14, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v30, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v28, v24, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v22, v35, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v21, v33, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v39, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v27, v37, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v31, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v29, v38, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v34, v22, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v48.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v49.l
; GFX11-TRUE16-NEXT: s_branch .LBB99_3
@@ -36386,62 +36469,62 @@ define inreg <16 x i16> @bitcast_v32i8_to_v16i16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB99_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v31
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v28
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v34
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v31
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v39
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v34
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v38
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v37
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v35
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v37
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v22, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v38
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v25, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v4, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v5, v26, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v32
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v3
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v5, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v30
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v35
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v33
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v29
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v30
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v27, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v3, v21, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v4, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v26
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v28
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v3, v20, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v4, v24, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v17
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v33
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v32
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v17
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v12, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v11, v8, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v13, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v15, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v20
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v21
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v20
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v18.l
; GFX11-TRUE16-NEXT: .LBB99_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -37675,158 +37758,162 @@ define <16 x half> @bitcast_v16bf16_to_v16f16(<16 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v16f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v7 :: v_dual_mov_b32 v15, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, v5 :: v_dual_mov_b32 v13, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v11, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v1 :: v_dual_mov_b32 v9, v0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB102_3
-; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
-; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB102_4
-; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB102_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, v9 :: v_dual_mov_b32 v1, v10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v11 :: v_dual_mov_b32 v3, v12
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, v13 :: v_dual_mov_b32 v5, v14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, v15 :: v_dual_mov_b32 v7, v16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
-; GFX11-TRUE16-NEXT: .LBB102_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v10
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v9
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v18, 0xffff0000, v7
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v8, 0x40c00000, v8 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v8, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v8, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v11, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v12, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v0, v5, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v12, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v9, v13, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v12
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v6, 0x40c00000, v6 :: v_dual_cndmask_b32 v9, v7, v9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v13, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v11, v12 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v10, 0x40c00000, v10 :: v_dual_add_f32 v11, 0x40c00000, v13
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v10
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v14, v10, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v12, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v12, v14, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v4 :: v_dual_add_f32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v11
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v12
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v15, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v12, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v11, v14, v15 :: v_dual_add_f32 v12, 0x40c00000, v12
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v15, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v12, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v12, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v6, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v9.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v5, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v8, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v11
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v13, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v15
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v12, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v13, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v14, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v14, v16, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v17, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v12 :: v_dual_lshlrev_b32 v6, 16, v13
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v8, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v7, 0x40c00000, v7 :: v_dual_lshlrev_b32 v14, 16, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v10.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v4, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v12, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v16, v17, v13, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v13
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v14, v15, vcc_lo
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v14, 0x40c00000, v14 :: v_dual_cndmask_b32 v13, v16, v17
+; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v12, v6, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v6
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
-; GFX11-TRUE16-NEXT: v_add3_u32 v12, v13, v7, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v13, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v5, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v5, v12, v14 :: v_dual_lshlrev_b32 v12, 16, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v14, 16, v16
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v12, 0x40c00000, v12
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v14, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v13, v6, v15 :: v_dual_and_b32 v16, 0xffff0000, v16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v15, v18, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v14, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v12, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v16, 0x40c00000, v16
-; GFX11-TRUE16-NEXT: v_add3_u32 v18, v18, v14, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v12, v6, v20, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v14, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v13
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v15, v16, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v16, v17, v14, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v14
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_bfe_u32 v21, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v18, v22, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v15, 0x40c00000, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v14, v16, v17, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v16, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v15, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v13.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v14
+; GFX11-TRUE16-NEXT: v_add3_u32 v16, v16, v7, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v12.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v6, 0x40c00000, v6
+; GFX11-TRUE16-NEXT: v_bfe_u32 v18, v6, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v17, v18, v6, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v18, v19, v15, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v19, 0x400000, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v15, v18, v19, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v17, v20, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v20, v21, v16, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v21, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v15, v19, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v12.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v20, v21, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v14.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v16, v21, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v14.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v15.l
+; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -39344,95 +39431,109 @@ define <32 x i8> @bitcast_v16f16_to_v32i8(<16 x half> %a, i32 %b) #0 {
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, v5 :: v_dual_mov_b32 v26, v4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v3 :: v_dual_mov_b32 v18, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v10, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v8
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB104_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB104_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v39, 0x200, v39 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v37, 0x200, v37 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v35, 0x200, v35 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v33, 0x200, v33 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v32, 0x200, v32 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v27, 0x200, v27 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v26, 0x200, v26 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, v11 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v19, 0x200, v19 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v18, 0x200, v18 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v10, 0x200, v10 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[34:35], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[35:36], 24, v[26:27]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v34, 0x200, v34 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v36, 0x200, v36 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v38, 0x200, v38 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[18:19]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[10:11]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v27
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v26
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v10
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB104_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v18.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v19.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v27.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v32.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v34.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v33.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16f16_to_v32i8:
@@ -40127,46 +40228,47 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v34, s16
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v24, s8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v8, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v32, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v36, s2
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v32, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v38, s0
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s6 :: v_dual_mov_b32 v20, s17
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s8
; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s6 :: v_dual_mov_b32 v33, s19
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v19, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s0
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v33, s19
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s17
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v39, s1
; GFX11-TRUE16-NEXT: s_branch .LBB105_3
; GFX11-TRUE16-NEXT: .LBB105_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
@@ -40184,45 +40286,45 @@ define inreg <32 x i8> @bitcast_v16f16_to_v32i8_scalar(<16 x half> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v12, 0x200, s3 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v20, 0x200, s17 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v39, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v37, 0x200, s3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v35, 0x200, s17 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v33, 0x200, s19 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v32, 0x200, s18 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v19, 0x200, s16 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v11, 0x200, s2 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v34, 0x200, s16 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v36, 0x200, s2 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v38, 0x200, s0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB105_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v24.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -40984,44 +41086,29 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v16f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: scratch_load_b32 v1, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v23.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v27.h
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v31.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v37
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB106_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -41031,133 +41118,133 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB106_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v20, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v28, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v31, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v37, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v14, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v31, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v12, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v12, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v34, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v8.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v8, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v26.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v10.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v34, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v13.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v9.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v36, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v12.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v11.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB106_2
; GFX11-TRUE16-NEXT: .LBB106_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v10.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v13.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v22.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v10.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v12.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v15.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.h, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v31.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v32.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v12.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v10.h, v1.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v10.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v15.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v20.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v31.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v11.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v14.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v8.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.l, v1.h
@@ -41987,63 +42074,65 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v11
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v10
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, s4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v19, s6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v31, s7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s4 :: v_dual_mov_b32 v31, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v22, s6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v34, s7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s8 :: v_dual_mov_b32 v25, s4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s5 :: v_dual_mov_b32 v27, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v25, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s6 :: v_dual_mov_b32 v38, s7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s8
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s17 :: v_dual_mov_b32 v12, s3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v14
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v38, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, s4 :: v_dual_mov_b32 v30, s8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v36, s5 :: v_dual_mov_b32 v39, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v35, s28
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s8 :: v_dual_mov_b32 v21, s27
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, s29 :: v_dual_mov_b32 v29, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s25 :: v_dual_mov_b32 v8, s23
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s24 :: v_dual_mov_b32 v24, s21
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s22 :: v_dual_mov_b32 v30, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s27 :: v_dual_mov_b32 v19, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v21, s29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s24 :: v_dual_mov_b32 v33, s28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s23 :: v_dual_mov_b32 v23, s25
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s21 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v28, s20
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s19 :: v_dual_mov_b32 v15, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s16 :: v_dual_mov_b32 v18, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s1 :: v_dual_mov_b32 v17, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s16 :: v_dual_mov_b32 v14, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s2 :: v_dual_mov_b32 v16, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v18, v12, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v17, v12, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v15, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v11, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v29, v21, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v20, v32, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v26, v38, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v48, v25, v34, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v49, v16, v28, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v17, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v26, v20, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v19, v30, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v27, v37, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v48, v25, v35, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v49, v18, v31, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v16, v14, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v30, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v28, v24, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v22, v35, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v21, v33, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v39, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v27, v37, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v31, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v29, v38, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v34, v22, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v48.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v49.l
; GFX11-TRUE16-NEXT: s_branch .LBB107_3
@@ -42057,62 +42146,62 @@ define inreg <16 x half> @bitcast_v32i8_to_v16f16_scalar(<32 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB107_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v31
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v28
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v34
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v31
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v39
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v34
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v38
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v37
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v35
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v37
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v22, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v38
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v25, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v4, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v5, v26, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v32
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v3
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v5, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v30
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v35
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v33
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v29
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v30
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v27, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v3, v21, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v4, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v26
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v28
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v3, v20, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v4, v24, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v17
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v33
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v32
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v17
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v12, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v11, v8, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v13, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v15, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v20
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v21
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v20
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v18.l
; GFX11-TRUE16-NEXT: .LBB107_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -42955,246 +43044,247 @@ define <32 x i8> @bitcast_v16bf16_to_v32i8(<16 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v16bf16_to_v32i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, v7 :: v_dual_mov_b32 v32, v6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v35, v5 :: v_dual_mov_b32 v34, v4
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, v3 :: v_dual_mov_b32 v36, v2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v39, v1 :: v_dual_mov_b32 v38, v0
; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[6:7]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[37:38], 24, v[4:5]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[2:3]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[48:49], 24, v[0:1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.h, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v3.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.h, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v5.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.h, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.h, v6.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v7.h
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v38
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB108_2: ; %Flow
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v39
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v37
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v32
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v13, 0x40c00000, v10 :: v_dual_add_f32 v8, 0x40c00000, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v11, v8, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v9, 0x40c00000, v9 :: v_dual_add_f32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v17, v10, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v11, v14, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v39
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v5, v7 :: v_dual_lshlrev_b32 v3, 16, v38
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v11, v15, v9, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v27, v17, v12, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v37
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v38
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v9, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v7, v5 :: v_dual_and_b32 v9, 0xffff0000, v34
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v36
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v1, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v3, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v36
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v2.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v12, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v26, v11, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v14, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v10.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v9, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v3, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v12, v13, 0x7fff
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 8, v27
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v11.h
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v12, v0, v3 :: v_dual_and_b32 v3, 0xffff0000, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v14
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v5, v8 :: v_dual_and_b32 v5, 0xffff0000, v35
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v14.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v33, v8, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v9, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v12.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[48:49], 24, v[26:27]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 8, v26
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v32, v1, v2 :: v_dual_add_f32 v1, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v13, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v7, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v35, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v9 :: v_dual_add_f32 v1, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v14, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v20, v9, v13, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v35
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v14, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff0000, v6
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v8 :: v_dual_add_f32 v5, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v33
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v34, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v6
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v7, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v28, v1, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v20.h
-; GFX11-TRUE16-NEXT: v_add3_u32 v1, v13, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v16.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v24, v6, v14, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v10.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v34
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v22.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v33
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v51, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v33
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v8, 0xffff0000, v32
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v8, 0x40c00000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v9, 0x40c00000, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add3_u32 v3, v4, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v11, v8, 0x7fff
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v1, v9, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v28.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 24, v27
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v50, v1, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v24.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[36:37], 24, v[50:51]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[37:38], 24, v[34:35]
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[38:39], 24, v[32:33]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v50
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v18.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v12, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v30.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v26.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v11.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v26.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v48.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v10.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v27.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v32.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v12.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v33.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v16.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v34.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v20.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v35.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v24.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v50.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v28.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v51.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v32i8:
@@ -44267,53 +44357,54 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v10, s6
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v24, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v34, s16
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v32, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s6 :: v_dual_mov_b32 v36, s2
; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s8 :: v_dual_mov_b32 v1, s6
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s5 :: v_dual_mov_b32 v32, s6
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v38, s0
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[18:19], 24
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s5
; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, s6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s19 :: v_dual_mov_b32 v27, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s17 :: v_dual_mov_b32 v19, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v11, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v19, s8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[8:9], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v11, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v33, s19
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s17
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v39, s1
; GFX11-TRUE16-NEXT: s_branch .LBB109_3
; GFX11-TRUE16-NEXT: .LBB109_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
@@ -44333,179 +44424,177 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s4
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: s_and_b32 s1, s3, 0xffff0000
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, 0xffff0000
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s1
; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v6 :: v_dual_add_nc_u32 v9, v9, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v3
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, 0xffff0000
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v7, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v4, v8 :: v_dual_add_nc_u32 v2, v2, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v9
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v39, 16, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v7
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.h, v6.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v38, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s2, 16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v10, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v2.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v4, 16, 1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v10, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: s_and_b32 s0, s17, 0xffff0000
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s17, 16
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v10
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v10, v10
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v7, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v8, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 16, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_and_b32 s0, s16, 0xffff0000
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v13, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v8 :: v_dual_add_nc_u32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s16, 16
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v8, v9
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v5, v13
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.h, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v10.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v5, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v1, v9 :: v_dual_add_nc_u32 v0, v3, v11
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v11
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s19, 0xffff0000
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v13
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v16, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v7
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s19, 0xffff0000
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s19, 16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v20, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v16, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v15, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v14.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v8, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v9, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.h, v22.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v8, v16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v5, v9
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_and_b32 s0, s18, 0xffff0000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v22.l
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s18, 16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v17, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v9, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, v15, v8
-; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v17, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, v19, v9
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s18, 16
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v7, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v12, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v1, v8 :: v_dual_add_nc_u32 v8, v11, v5
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, v13, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v12, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v20
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v15, v17
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v20
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v11
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v9, v12
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v7
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v13
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v9
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v0, v7 :: v_dual_add_nc_u32 v7, 0x7fff, v16
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v17
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v18.l
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v13, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[24:25], 24, v[19:20]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v15, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v17, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v12
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v28, 16, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v16, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.h, v30.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[16:17], 24, v[11:12]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v26.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v28
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[32:33], 24, v[27:28]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v27
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.h, v18.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[34:35]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v34
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.h, v30.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[36:37]
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[38:39]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v37
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.h, v26.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v33
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v36
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[27:28], 24, v[32:33]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v32
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v39
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v38
; GFX11-TRUE16-NEXT: .LBB109_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v38.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v39.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v34.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v33.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v16bf16_to_v32i8_scalar:
@@ -45413,44 +45502,29 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v32i8_to_v16bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: scratch_load_b32 v1, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v27.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v17.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v19.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v23.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v25.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v27.h
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v13.h, 8, v0.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v31.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v37
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB110_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -45460,133 +45534,133 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB110_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v13, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v32, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v11, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v20, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v28, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v31, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v37, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v14, v15, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v31, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v22.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v12, v13, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v6.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v30.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v15, v12, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v34, v35, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v8.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v8, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v5.l, v9.h
; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v24.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v26.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v10.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v34, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v18.l
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v6.l, v13.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v9.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v36, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v7.l, v12.h
+; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v7.h, v11.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v4.l, v10.h
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v4.h, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v8.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB110_2
; GFX11-TRUE16-NEXT: .LBB110_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v28.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v29.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v20.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v10.h, v0.h
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v13.h, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v22.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v10.l, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v25.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v12.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v15.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v8.h, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.h, v1.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v31.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v9.l, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v32.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.l, v0.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v12.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v10.h, v1.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v14.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v8.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v13.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v9.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.h, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v28.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v10.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v15.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v34.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v23.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v20.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, v31.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v17.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v11.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v11.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.l, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v13.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v36.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v12.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v35.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v8.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v14.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v8.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v9.l, v1.h
@@ -46408,63 +46482,65 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v11
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v10
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, s4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v28, s5 :: v_dual_mov_b32 v19, s6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v31, s7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s4 :: v_dual_mov_b32 v31, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v22, s6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v34, s7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s8 :: v_dual_mov_b32 v25, s4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v34, s5 :: v_dual_mov_b32 v27, s6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v25, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s6 :: v_dual_mov_b32 v38, s7
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s8
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v1
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s17 :: v_dual_mov_b32 v12, s3
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v14
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v38, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v37, s4 :: v_dual_mov_b32 v30, s8
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v36, s5 :: v_dual_mov_b32 v39, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s7 :: v_dual_mov_b32 v35, s28
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s8 :: v_dual_mov_b32 v21, s27
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v22, s29 :: v_dual_mov_b32 v29, s26
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v23, s25 :: v_dual_mov_b32 v8, s23
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s24 :: v_dual_mov_b32 v24, s21
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s22 :: v_dual_mov_b32 v30, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v20, s27 :: v_dual_mov_b32 v19, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s26 :: v_dual_mov_b32 v21, s29
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s24 :: v_dual_mov_b32 v33, s28
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s23 :: v_dual_mov_b32 v23, s25
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s21 :: v_dual_mov_b32 v11, s22
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v28, s20
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s19 :: v_dual_mov_b32 v15, s18
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s16 :: v_dual_mov_b32 v18, s2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s1 :: v_dual_mov_b32 v17, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s16 :: v_dual_mov_b32 v14, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s2 :: v_dual_mov_b32 v16, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v18, v12, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v17, v12, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v15, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v11, v8, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v29, v21, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v20, v32, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v26, v38, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v48, v25, v34, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v49, v16, v28, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v17, v14, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v26, v20, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v19, v30, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v27, v37, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v48, v25, v35, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v49, v18, v31, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v16, v14, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v13, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v30, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v33, v23, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v28, v24, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v32, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v22, v35, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v21, v33, 0x6050400
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v39, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v27, v37, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v31, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v29, v38, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v34, v22, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v48.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v49.l
; GFX11-TRUE16-NEXT: s_branch .LBB111_3
@@ -46478,62 +46554,62 @@ define inreg <16 x bfloat> @bitcast_v32i8_to_v16bf16_scalar(<32 x i8> inreg %a,
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB111_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v31
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v28
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v34
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v31
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v39
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v34
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v38
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v37
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v35
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 3, v37
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v22, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v38
; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v25, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v0
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v4, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v5, v26, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v32
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v3
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v5, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v30
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v35
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v33
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v29
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v30
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v27, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v3, v21, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v21, v4, v24, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v22, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v26
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v28
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v3, v20, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v4, v24, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v18
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v17
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v33
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v32
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 3, v13
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v17
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 3, v16
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v12, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v10, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v11, v8, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v13, v9, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v10, v15, v14, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v20
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v19
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v8
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v21
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v20
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v9
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v10
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v12.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v21.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v22.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v18.l
; GFX11-TRUE16-NEXT: .LBB111_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
index 94abad4a236ff..937cf1965ccca 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.320bit.ll
@@ -5119,31 +5119,32 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v10i32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v66
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -5154,11 +5155,11 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB14_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v65, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v55, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v53, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v51, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v49, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v12, v13, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v14, v15, 0xc0c0004
@@ -5175,26 +5176,26 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v37, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v64, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v55, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v52, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v51, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v14, 16, v13
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
@@ -5216,39 +5217,39 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB14_2
; GFX11-TRUE16-NEXT: .LBB14_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v55.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v51.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v10.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v13.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v31.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v12.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v14.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v16.l, 3
@@ -5284,18 +5285,18 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v55.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v5.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v51.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v54.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v6.h
@@ -5304,12 +5305,12 @@ define <10 x i32> @bitcast_v40i8_to_v10i32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v64.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v11.l, v9.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v11.h, v9.h
@@ -12406,31 +12407,32 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v10f32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v66, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v49, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v51, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v50, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v52, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v53, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v54, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v66
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -12441,11 +12443,11 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB34_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v65, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v55, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v53, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v51, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v49, v48, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v36, v35, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v34, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v32, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v11, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v12, v13, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v7, v14, v15, 0xc0c0004
@@ -12462,26 +12464,26 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v37, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v64, v54, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v55, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v52, v50, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v51, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v14, 16, v13
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr12_lo16
@@ -12503,39 +12505,39 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB34_2
; GFX11-TRUE16-NEXT: .LBB34_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v55.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v36.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v51.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v34.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v49.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v32.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v10.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v13.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v48.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v31.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v12.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v14.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v16.l, 3
@@ -12571,18 +12573,18 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v55.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v5.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v51.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v54.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v50.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v6.h
@@ -12591,12 +12593,12 @@ define <10 x float> @bitcast_v40i8_to_v10f32(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v64.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v49.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v11.l, v9.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v11.h, v9.h
@@ -19356,57 +19358,42 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v20i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v2, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v21, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v18, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v20, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v1.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v33.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v34.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v70
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB50_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -19417,173 +19404,174 @@ define <20 x i16> @bitcast_v40i8_to_v20i16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB50_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v24, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v50, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v66, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v22, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v51, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v49, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v39, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v37, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v67, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v24, v25, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v26, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v38, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v55, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v21.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v33, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v22, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v31.h
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v18.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v53, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v34, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v30, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v19.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v48, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v35, v52, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v54, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v64, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v16.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v69, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v17.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v65, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB50_2
; GFX11-TRUE16-NEXT: .LBB50_4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v65.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.h, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v20.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v18.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v64.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v16.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v19.h, v0.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v28.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v17.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v25.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v55.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v49.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v28.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v19.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v22.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v24.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v50.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v20.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v10.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.l, v1.h
@@ -20637,92 +20625,90 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v21
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v16
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s3 :: v_dual_mov_b32 v28, s5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v29, s6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v18
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v20, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s4 :: v_dual_mov_b32 v35, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v19
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v12
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v53, s5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v30, s4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s17 :: v_dual_mov_b32 v20, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, s7 :: v_dual_mov_b32 v34, s4
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s8 :: v_dual_mov_b32 v48, s6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v69, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v25, s5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v66, s5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s29 :: v_dual_mov_b32 v31, s25
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v64, s7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v70, s4
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v32, s6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v12
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v52, s7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v22
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v64, s5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v65, s8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v68, s5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v34, s29
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v70, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v66, s28 :: v_dual_mov_b32 v39, s8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v67, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v67, s7 :: v_dual_mov_b32 v50, s6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v36, s8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v4
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v49, s6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v54, s8 :: v_dual_mov_b32 v33, s27
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v50, s26
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v54, s5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v68, s8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v69, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v71, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v65, s5 :: v_dual_mov_b32 v28, s7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v55, s28
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v39, s8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s27 :: v_dual_mov_b32 v38, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v53, s24 :: v_dual_mov_b32 v16, s23
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s7 :: v_dual_mov_b32 v71, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v36, s25 :: v_dual_mov_b32 v55, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s23 :: v_dual_mov_b32 v19, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v38, s21 :: v_dual_mov_b32 v51, s20
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s19 :: v_dual_mov_b32 v17, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v23, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s2 :: v_dual_mov_b32 v26, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v22, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s22 :: v_dual_mov_b32 v52, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s21 :: v_dual_mov_b32 v48, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s19 :: v_dual_mov_b32 v23, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s16 :: v_dual_mov_b32 v22, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s2 :: v_dual_mov_b32 v24, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v27, v21, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v24, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v25, v20, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v18, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v19, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v50, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v54, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v70, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v64, v37, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v48, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v26, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v23, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v29, v35, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v20, v28, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v38, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v52, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v69, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v54, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v50, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v24, v22, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v32, v51, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v26, v35, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v55, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v48, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v53, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v66, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v30, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v67, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v65, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v68, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v31, v69, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v53, v30, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v36, v70, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v66, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v11.l
; GFX11-TRUE16-NEXT: s_branch .LBB51_3
@@ -20736,71 +20722,71 @@ define inreg <20 x i16> @bitcast_v40i8_to_v20i16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v28
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v65
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v35
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v30, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v31, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v35
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v68
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v51
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v70
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v26, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v48
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v50
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v67
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v64
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v70
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v69
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v71
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v54
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v55
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v52
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v54
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v55
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v51
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v48
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v30, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v50
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v4, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v38
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v4, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v27
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v24
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v25
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v23
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v19
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v3, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v20
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v23
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 3, v26
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v21
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 3, v24
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v3, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v18, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v19, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v20, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v21, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v23, v22, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v26
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v17
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v18
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v20.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v16.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v12.l
@@ -26005,57 +25991,42 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v20f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v32, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_b32 v70, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v33, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v34, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v25.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v24.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v22.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v21.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v9.l
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v0, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_b32 v2, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v0, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v1, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v21, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v18, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v20, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v3.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(7)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v26.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v19.h, 8, v0.l
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(6)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v22.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.h, 8, v1.l
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v17.l, 8, v32.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.l, 8, v33.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v16.h, 8, v34.h
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v70
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB62_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -26066,173 +26037,174 @@ define <20 x half> @bitcast_v40i8_to_v20f16(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB62_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v24, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v30, v23, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v50, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v66, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v22, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v51, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v18, v19, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v49, v32, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v33, v25, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v39, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v37, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v67, v65, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v24, v25, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v32.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v26, v19, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v38, v29, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v55, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v21.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v33, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v22, v23, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v31.h
-; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v31.l
+; GFX11-TRUE16-NEXT: v_and_b16 v8.h, 0xff, v20.h
+; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v18.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v53, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v17.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v34, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v30, v55, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v10.l, v8.l, v19.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v48, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v35, v52, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v54, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v20, v21, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v64, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.l
-; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v16.h
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v69, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v28, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v16.h
+; GFX11-TRUE16-NEXT: v_or_b16 v8.h, v9.l, v17.h
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v65, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v11.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr22_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr23_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr24_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr25_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr26_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr27_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr20_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr21_hi16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_hi16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_hi16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_hi16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB62_2
; GFX11-TRUE16-NEXT: .LBB62_4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v69.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v65.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v21.h, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v31.h, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v31.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v20.h, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v64.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v18.h, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v26.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v17.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v64.l, 3
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v16.h, v2.l
+; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v19.h, v0.h
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v28.l, 3
+; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v16.h, v1.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v17.h, v2.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v52.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v53.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v29.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.l, v30.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.l, 0x300, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v24.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v8.h, 0x300, v2.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v65.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v25.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v20.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v55.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v27.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v39.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v21.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v3.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v49.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v23.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v50.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, 0x300, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.h, v1.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v28.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v19.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v35.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v33.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v2.l, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v39.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v27.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v52.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v29.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v38.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v25.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v36.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.l, 0x300, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v2.h, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v3.l, v1.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v30.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v51.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, 0x300, v0.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v22.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v49.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, 0x300, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v24.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v50.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, 0x300, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v23.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v21.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v19.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v53.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v18.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v20.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v37.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.l, v0.h
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v10.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v11.l, v1.h
@@ -27286,92 +27258,90 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v21
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v20
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v16
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v17
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s3 :: v_dual_mov_b32 v28, s5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v29, s6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v18
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v20, s4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v26, s4 :: v_dual_mov_b32 v35, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v19
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v35, s7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v12
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v53, s5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v30, s4
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, s17 :: v_dual_mov_b32 v20, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v51, s7 :: v_dual_mov_b32 v34, s4
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v31, s8 :: v_dual_mov_b32 v48, s6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v11
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v10
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v69, s4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v25, s5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v66, s5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v13
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v30, s29 :: v_dual_mov_b32 v31, s25
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v64, s7
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v6
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v70, s4
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v9
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v32, s6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v12
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v27, s5
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v8
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v52, s7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v7
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v6
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s3, v22
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v64, s5
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v37, s4
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v4
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v65, s8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v68, s5
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v34, s29
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v70, s4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v66, s28 :: v_dual_mov_b32 v39, s8
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v67, s7
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v67, s7 :: v_dual_mov_b32 v50, s6
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v36, s8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v10
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v4
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s7, v1
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v49, s6
; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s6, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v54, s8 :: v_dual_mov_b32 v33, s27
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v50, s26
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v54, s5
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s5, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v68, s8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v5
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v69, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v71, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v65, s5 :: v_dual_mov_b32 v28, s7
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v55, s28
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v39, s8
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s8, v0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v29, s27 :: v_dual_mov_b32 v38, s26
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v53, s24 :: v_dual_mov_b32 v16, s23
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v32, s7 :: v_dual_mov_b32 v71, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v36, s25 :: v_dual_mov_b32 v55, s24
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, s23 :: v_dual_mov_b32 v19, s22
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v38, s21 :: v_dual_mov_b32 v51, s20
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s19 :: v_dual_mov_b32 v17, s17
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v24, s18 :: v_dual_mov_b32 v23, s16
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v27, s2 :: v_dual_mov_b32 v26, s0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v22, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, s22 :: v_dual_mov_b32 v52, s8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v33, s21 :: v_dual_mov_b32 v48, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v18, s19 :: v_dual_mov_b32 v23, s18
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v21, s16 :: v_dual_mov_b32 v22, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v25, s2 :: v_dual_mov_b32 v24, s0
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB63_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v27, v21, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v24, v18, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v25, v20, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v23, v18, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v19, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v50, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v54, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v70, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v64, v37, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v48, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v26, v22, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v23, v17, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v29, v35, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v20, v28, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v38, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v52, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v69, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v54, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v50, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v24, v22, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v21, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v32, v51, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v26, v35, 0x6050400
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v38, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v55, v36, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v48, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v53, v31, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v66, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v30, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v71, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v6.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v7.l
; GFX11-TRUE16-NEXT: v_perm_b32 v6, v67, v49, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v65, v52, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v68, v64, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v9.l
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v31, v69, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v9, v53, v30, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v36, v70, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v66, v34, 0xc0c0004
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v11.l
; GFX11-TRUE16-NEXT: s_branch .LBB63_3
@@ -27385,71 +27355,71 @@ define inreg <20 x half> @bitcast_v40i8_to_v20f16_scalar(<40 x i8> inreg %a, i32
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB63_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v53
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v28
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v69
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v65
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v35
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v30, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v20, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v31, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v52, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v35
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v68
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v51
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v70
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v26, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v36, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v48
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v50
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v67
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x300, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v64
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v70
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v25, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v69
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 0x300, v3
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v27, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v49, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v71
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x300, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v12, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v66
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v54
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v55
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v52
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v54
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v65, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, 0x300, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v55
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v51
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v34, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v32, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v48
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v30, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v28, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v37, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v50
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v20, v4, v38, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v38
+; GFX11-TRUE16-NEXT: v_perm_b32 v26, v4, v33, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v25, 0x300, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v27
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v24
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v27, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v25
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v23
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 3, v19
-; GFX11-TRUE16-NEXT: v_perm_b32 v15, v3, v33, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v20
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 3, v23
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 3, v26
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v21, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v53
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v21, 3, v21
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v23, 3, v24
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v3, v29, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v20, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v18, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v16, v19, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v17, v20, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v21, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v18, v23, v22, 0xc0c0004
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, 0x300, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v19, 0x300, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v20, 0x300, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x300, v16
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v26
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v17
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v18
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v19.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v20.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v16.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v25.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v27.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v14.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v13.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v12.l
@@ -30517,38 +30487,38 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v5f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v80
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB72_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -30559,15 +30529,15 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB72_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v71, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v69, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v67, v66, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v65, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v53, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v52, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v49, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v48, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v36, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v35, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v18, v19, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
@@ -30580,33 +30550,33 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v37, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v69, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v66, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v14, 16, v13
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
@@ -30621,46 +30591,46 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB72_2
; GFX11-TRUE16-NEXT: .LBB72_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v71.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v69.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v66.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v53.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v52.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v35.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v23.l
@@ -30668,7 +30638,7 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v3.h
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
@@ -30689,18 +30659,18 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v5.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v65.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v67.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v6.h
@@ -30709,12 +30679,12 @@ define <5 x double> @bitcast_v40i8_to_v5f64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v11.l, v9.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v11.h, v9.h
@@ -34758,38 +34728,38 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-LABEL: bitcast_v40i8_to_v5i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v0.l
; GFX11-TRUE16-NEXT: s_clause 0x9
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v31, off, s32 offset:32
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v33, off, s32 offset:28
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v32, off, s32 offset:24
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v34, off, s32 offset:20
-; GFX11-TRUE16-NEXT: scratch_load_b32 v80, off, s32 offset:36
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v35, off, s32 offset:16
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v37, off, s32 offset:12
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v36, off, s32 offset:8
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v38, off, s32 offset:4
-; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v39, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v11.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v10.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v54.l, v9.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v55.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v64.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v65.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v70.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v71.l, v0.l
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v55, off, s32 offset:32
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v65, off, s32 offset:28
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v64, off, s32 offset:24
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v66, off, s32 offset:20
+; GFX11-TRUE16-NEXT: scratch_load_b32 v0, off, s32 offset:36
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v67, off, s32 offset:16
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v69, off, s32 offset:12
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v68, off, s32 offset:8
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v70, off, s32 offset:4
+; GFX11-TRUE16-NEXT: scratch_load_d16_b16 v71, off, s32
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v15.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v12.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v11.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v10.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v37.l, v9.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v39.l, v7.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v52.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v53.l, v1.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(5)
-; GFX11-TRUE16-NEXT: v_cmpx_ne_u32_e32 0, v80
+; GFX11-TRUE16-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15
+; GFX11-TRUE16-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB76_3
; GFX11-TRUE16-NEXT: ; %bb.1: ; %Flow
@@ -34800,15 +34770,15 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB76_3: ; %cmp.false
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v71, v70, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v69, v68, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v67, v66, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v3, v65, v64, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v54, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v5, v53, v51, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v52, v50, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v7, v49, v16, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v8, v48, v17, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v54, v53, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v52, v51, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v50, v49, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v48, v39, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v38, v37, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v36, v34, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v35, v33, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v32, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v16, v17, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v18, v19, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v1, v3, 16, v2
@@ -34821,33 +34791,33 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_perm_b32 v8, v26, v27, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v9, v28, v29, 0xc0c0004
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v39, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v11, v38, v36, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v12, v37, v35, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v13, v34, v32, 0xc0c0004
-; GFX11-TRUE16-NEXT: v_perm_b32 v14, v33, v31, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v30, v71, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v70, v68, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v69, v67, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v66, v64, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v65, v55, 0xc0c0004
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v5, v6, 16, v5
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v6, v8, 16, v7
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v7, v10, 16, v9
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v8, v12, 16, v11
; GFX11-TRUE16-NEXT: v_lshl_or_b32 v9, v14, 16, v13
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr54_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr53_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr52_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr51_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr50_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr49_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr48_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr16_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr17_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr18_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr19_lo16
@@ -34862,46 +34832,46 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr28_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr29_lo16
; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr30_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr39_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr38_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr36_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr37_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr35_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr34_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr32_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr33_lo16
-; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr31_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr71_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr70_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr68_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr69_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr67_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr66_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr64_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr65_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr55_lo16
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB76_2
; GFX11-TRUE16-NEXT: .LBB76_4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v71.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v69.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v70.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v68.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v67.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v54.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v52.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v53.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v51.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v50.l, 3
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v65.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v66.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v64.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v49.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v39.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v1.l, v0.l
; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v1.h, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v55.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v53.l, 3
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v50.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.l, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.l, v36.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v33.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v2.h, v1.l
; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v3.h, v1.h
; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v54.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v52.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v49.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v48.l, 3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v37.l
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v3.h, v35.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v4.h, v32.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.l, v16.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v18.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.h, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v51.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v34.l
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v3.h
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, v22.l, 3
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v23.l
@@ -34909,7 +34879,7 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v3.l, v2.h
; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v4.l, v3.h
; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v16.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v31.l
; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v5.l
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v17.l
; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v6.l
@@ -34930,18 +34900,18 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v29.l
; GFX11-TRUE16-NEXT: v_or_b16 v7.l, v7.h, v7.l
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(3)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v37.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.l, v69.l, 3
; GFX11-TRUE16-NEXT: v_or_b16 v5.l, v5.h, v5.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v5.h, v26.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v34.l, 3
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v33.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v9.h, v66.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v10.h, v65.l, 3
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v36.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v68.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.l, 0xff, v9.l
; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v35.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.l, 8, v67.l
; GFX11-TRUE16-NEXT: v_and_b16 v9.h, 0xff, v9.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v32.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v11.h, 8, v64.l
; GFX11-TRUE16-NEXT: v_and_b16 v10.h, 0xff, v10.h
; GFX11-TRUE16-NEXT: v_or_b16 v7.h, v8.l, v5.h
; GFX11-TRUE16-NEXT: v_or_b16 v8.l, v8.h, v6.h
@@ -34950,12 +34920,12 @@ define <5 x i64> @bitcast_v40i8_to_v5i64(<40 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.l, 0x300, v7.l
; GFX11-TRUE16-NEXT: v_add_nc_u16 v6.h, 0x300, v7.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v38.l, 3
+; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.h, v70.l, 3
; GFX11-TRUE16-NEXT: v_add_nc_u16 v7.l, 0x300, v8.l
; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v8.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v39.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v31.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v71.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v12.l, 8, v55.l
; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v7.h
; GFX11-TRUE16-NEXT: v_or_b16 v9.l, v11.l, v9.l
; GFX11-TRUE16-NEXT: v_or_b16 v9.h, v11.h, v9.h
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 7522d1b697b33..bd4c23bf665ff 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -2350,35 +2350,38 @@ define inreg <4 x i8> @bitcast_i32_to_v4i8_scalar(i32 inreg %a, i32 inreg %b) #0
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB21_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
; GFX11-TRUE16-NEXT: s_branch .LBB21_3
; GFX11-TRUE16-NEXT: .LBB21_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: .LBB21_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB21_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: .LBB21_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i32_to_v4i8_scalar:
@@ -2783,46 +2786,88 @@ define inreg i32 @bitcast_v4i8_to_i32_scalar(<4 x i8> inreg %a, i32 inreg %b) #0
; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4i8_to_i32_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_branch .LBB23_3
-; GFX11-NEXT: .LBB23_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: .LBB23_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB23_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB23_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4i8_to_i32_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB23_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT: s_branch .LBB23_3
+; GFX11-TRUE16-NEXT: .LBB23_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB23_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: .LBB23_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4i8_to_i32_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB23_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB23_3
+; GFX11-FAKE16-NEXT: .LBB23_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-FAKE16-NEXT: .LBB23_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: .LBB23_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -4857,39 +4902,74 @@ define inreg <4 x i8> @bitcast_f32_to_v4i8_scalar(float inreg %a, i32 inreg %b)
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_f32_to_v4i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-NEXT: s_branch .LBB41_3
-; GFX11-NEXT: .LBB41_2:
-; GFX11-NEXT: s_mov_b32 s3, -1
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr1
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: .LBB41_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s3, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_add_f32_e64 v0, s0, 1.0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB41_5:
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_f32_to_v4i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB41_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX11-TRUE16-NEXT: s_branch .LBB41_3
+; GFX11-TRUE16-NEXT: .LBB41_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: .LBB41_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, s0, 1.0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: .LBB41_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_f32_to_v4i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB41_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB41_3
+; GFX11-FAKE16-NEXT: .LBB41_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: .LBB41_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, s0, 1.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-NEXT: .LBB41_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5259,46 +5339,88 @@ define inreg float @bitcast_v4i8_to_f32_scalar(<4 x i8> inreg %a, i32 inreg %b)
; GFX9-NEXT: .LBB43_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4i8_to_f32_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB43_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_branch .LBB43_3
-; GFX11-NEXT: .LBB43_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: .LBB43_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB43_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB43_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4i8_to_f32_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB43_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT: s_branch .LBB43_3
+; GFX11-TRUE16-NEXT: .LBB43_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB43_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: .LBB43_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4i8_to_f32_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB43_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB43_3
+; GFX11-FAKE16-NEXT: .LBB43_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-FAKE16-NEXT: .LBB43_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB43_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: .LBB43_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7017,39 +7139,74 @@ define inreg <4 x i8> @bitcast_v2i16_to_v4i8_scalar(<2 x i16> inreg %a, i32 inre
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v2i16_to_v4i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB57_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-NEXT: s_branch .LBB57_3
-; GFX11-NEXT: .LBB57_2:
-; GFX11-NEXT: s_mov_b32 s3, -1
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr1
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: .LBB57_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s3, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB57_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB57_5:
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v2i16_to_v4i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB57_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX11-TRUE16-NEXT: s_branch .LBB57_3
+; GFX11-TRUE16-NEXT: .LBB57_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: .LBB57_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: .LBB57_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v2i16_to_v4i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB57_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB57_3
+; GFX11-FAKE16-NEXT: .LBB57_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: .LBB57_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB57_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v0, s0, 3 op_sel_hi:[1,0]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-NEXT: .LBB57_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -7425,46 +7582,88 @@ define inreg <2 x i16> @bitcast_v4i8_to_v2i16_scalar(<4 x i8> inreg %a, i32 inre
; GFX9-NEXT: .LBB59_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4i8_to_v2i16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB59_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_branch .LBB59_3
-; GFX11-NEXT: .LBB59_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: .LBB59_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB59_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB59_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4i8_to_v2i16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB59_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT: s_branch .LBB59_3
+; GFX11-TRUE16-NEXT: .LBB59_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB59_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: .LBB59_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4i8_to_v2i16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB59_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB59_3
+; GFX11-FAKE16-NEXT: .LBB59_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-FAKE16-NEXT: .LBB59_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB59_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: .LBB59_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -8805,39 +9004,74 @@ define inreg <4 x i8> @bitcast_v2f16_to_v4i8_scalar(<2 x half> inreg %a, i32 inr
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v2f16_to_v4i8_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB69_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: s_lshr_b32 s2, s0, 24
-; GFX11-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-NEXT: s_lshr_b32 s4, s0, 8
-; GFX11-NEXT: s_branch .LBB69_3
-; GFX11-NEXT: .LBB69_2:
-; GFX11-NEXT: s_mov_b32 s3, -1
-; GFX11-NEXT: ; implicit-def: $sgpr4
-; GFX11-NEXT: ; implicit-def: $sgpr1
-; GFX11-NEXT: ; implicit-def: $sgpr2
-; GFX11-NEXT: .LBB69_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s3, s3, exec_lo
-; GFX11-NEXT: s_cselect_b32 s3, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s3, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB69_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 24, v0
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-; GFX11-NEXT: .LBB69_5:
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v2f16_to_v4i8_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB69_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
+; GFX11-TRUE16-NEXT: s_branch .LBB69_3
+; GFX11-TRUE16-NEXT: .LBB69_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: .LBB69_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: .LBB69_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v2f16_to_v4i8_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s1, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB69_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-FAKE16-NEXT: s_branch .LBB69_3
+; GFX11-FAKE16-NEXT: .LBB69_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr4
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-FAKE16-NEXT: ; implicit-def: $sgpr2
+; GFX11-FAKE16-NEXT: .LBB69_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s3, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB69_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_pk_add_f16 v0, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-NEXT: .LBB69_5:
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -9213,46 +9447,88 @@ define inreg <2 x half> @bitcast_v4i8_to_v2f16_scalar(<4 x i8> inreg %a, i32 inr
; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4i8_to_v2f16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_branch .LBB71_3
-; GFX11-NEXT: .LBB71_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: .LBB71_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB71_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4i8_to_v2f16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT: s_branch .LBB71_3
+; GFX11-TRUE16-NEXT: .LBB71_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB71_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: .LBB71_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4i8_to_v2f16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB71_3
+; GFX11-FAKE16-NEXT: .LBB71_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-FAKE16-NEXT: .LBB71_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: .LBB71_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10263,27 +10539,31 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB77_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
; GFX11-TRUE16-NEXT: s_branch .LBB77_3
; GFX11-TRUE16-NEXT: .LBB77_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: .LBB77_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB77_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xffff0000
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s1
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
@@ -10301,19 +10581,14 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-; GFX11-TRUE16-NEXT: .LBB77_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v1, s3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; GFX11-TRUE16-NEXT: .LBB77_5: ; %end
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2bf16_to_v4i8_scalar:
@@ -10740,46 +11015,88 @@ define inreg <2 x bfloat> @bitcast_v4i8_to_v2bf16_scalar(<4 x i8> inreg %a, i32
; GFX9-NEXT: .LBB79_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4i8_to_v2bf16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB79_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_branch .LBB79_3
-; GFX11-NEXT: .LBB79_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: .LBB79_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB79_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB79_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4i8_to_v2bf16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB79_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT: s_branch .LBB79_3
+; GFX11-TRUE16-NEXT: .LBB79_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB79_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: .LBB79_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4i8_to_v2bf16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB79_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB79_3
+; GFX11-FAKE16-NEXT: .LBB79_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-FAKE16-NEXT: .LBB79_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB79_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: .LBB79_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11072,35 +11389,38 @@ define inreg <4 x i8> @bitcast_v1i32_to_v4i8_scalar(<1 x i32> inreg %a, i32 inre
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB81_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
; GFX11-TRUE16-NEXT: s_branch .LBB81_3
; GFX11-TRUE16-NEXT: .LBB81_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr1
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
; GFX11-TRUE16-NEXT: .LBB81_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s1, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s1, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB81_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: .LBB81_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v1i32_to_v4i8_scalar:
@@ -11505,46 +11825,88 @@ define inreg <1 x i32> @bitcast_v4i8_to_v1i32_scalar(<4 x i8> inreg %a, i32 inre
; GFX9-NEXT: .LBB83_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v4i8_to_v1i32_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s16, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB83_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: v_perm_b32 v0, s0, s1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_branch .LBB83_3
-; GFX11-NEXT: .LBB83_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0
-; GFX11-NEXT: .LBB83_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB83_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v0, s2, s3, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: .LBB83_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v4i8_to_v1i32_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v4, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB83_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX11-TRUE16-NEXT: s_branch .LBB83_3
+; GFX11-TRUE16-NEXT: .LBB83_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-TRUE16-NEXT: .LBB83_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: .LBB83_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v4i8_to_v1i32_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s16, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB83_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s0, s1, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB83_3
+; GFX11-FAKE16-NEXT: .LBB83_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0
+; GFX11-FAKE16-NEXT: .LBB83_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB83_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: .LBB83_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 8b1ab6672ae86..837ad20b843fa 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -2400,47 +2400,45 @@ define i64 @bitcast_v4bf16_to_i64(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB22_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v4, v5 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: .LBB22_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -2689,46 +2687,48 @@ define inreg i64 @bitcast_v4bf16_to_i64_scalar(<4 x bfloat> inreg %a, i32 inreg
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_nc_u32 v8, v8, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v5 :: v_dual_add_nc_u32 v5, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v5 :: v_dual_add_nc_u32 v9, v9, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB23_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -3178,48 +3178,55 @@ define inreg <8 x i8> @bitcast_i64_to_v8i8_scalar(i64 inreg %a, i32 inreg %b) #0
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB25_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_branch .LBB25_3
; GFX11-TRUE16-NEXT: .LBB25_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: .LBB25_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB25_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_u32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_addc_u32 s1, s1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s4
; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s0
; GFX11-TRUE16-NEXT: .LBB25_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_i64_to_v8i8_scalar:
@@ -3786,62 +3793,122 @@ define inreg i64 @bitcast_v8i8_to_i64_scalar(<8 x i8> inreg %a, i32 inreg %b) #0
; GFX9-NEXT: .LBB27_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8i8_to_i64_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB27_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_branch .LBB27_3
-; GFX11-NEXT: .LBB27_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: .LBB27_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB27_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB27_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8i8_to_i64_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB27_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
+; GFX11-TRUE16-NEXT: s_branch .LBB27_3
+; GFX11-TRUE16-NEXT: .LBB27_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-TRUE16-NEXT: .LBB27_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: .LBB27_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8i8_to_i64_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB27_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB27_3
+; GFX11-FAKE16-NEXT: .LBB27_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-FAKE16-NEXT: .LBB27_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB27_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: .LBB27_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5877,47 +5944,45 @@ define double @bitcast_v4bf16_to_f64(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB46_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v4, v5 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: .LBB46_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -6166,46 +6231,48 @@ define inreg double @bitcast_v4bf16_to_f64_scalar(<4 x bfloat> inreg %a, i32 inr
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_nc_u32 v8, v8, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v5 :: v_dual_add_nc_u32 v5, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v5 :: v_dual_add_nc_u32 v9, v9, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB47_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -6657,51 +6724,50 @@ define inreg <8 x i8> @bitcast_f64_to_v8i8_scalar(double inreg %a, i32 inreg %b)
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB49_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s1
; GFX11-TRUE16-NEXT: s_branch .LBB49_3
; GFX11-TRUE16-NEXT: .LBB49_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: .LBB49_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB49_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f64 v[3:4], s[0:1], 1.0
+; GFX11-TRUE16-NEXT: v_add_f64 v[8:9], s[0:1], 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB49_6
-; GFX11-TRUE16-NEXT: .LBB49_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v1, s7
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s2 :: v_dual_mov_b32 v6, s5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s6 :: v_dual_mov_b32 v8, s3
-; GFX11-TRUE16-NEXT: .LBB49_6: ; %end
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-TRUE16-NEXT: .LBB49_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_f64_to_v8i8_scalar:
@@ -7272,75 +7338,135 @@ define inreg double @bitcast_v8i8_to_f64_scalar(<8 x i8> inreg %a, i32 inreg %b)
; GFX9-NEXT: .LBB51_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8i8_to_f64_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB51_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_branch .LBB51_3
-; GFX11-NEXT: .LBB51_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: .LBB51_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB51_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB51_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
- %cmp = icmp eq i32 %b, 0
- br i1 %cmp, label %cmp.true, label %cmp.false
-
-cmp.true:
- %a1 = add <8 x i8> %a, splat (i8 3)
- %a2 = bitcast <8 x i8> %a1 to double
- br label %end
-
-cmp.false:
- %a3 = bitcast <8 x i8> %a to double
- br label %end
-
-end:
+; GFX11-TRUE16-LABEL: bitcast_v8i8_to_f64_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB51_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
+; GFX11-TRUE16-NEXT: s_branch .LBB51_3
+; GFX11-TRUE16-NEXT: .LBB51_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-TRUE16-NEXT: .LBB51_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: .LBB51_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8i8_to_f64_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB51_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB51_3
+; GFX11-FAKE16-NEXT: .LBB51_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-FAKE16-NEXT: .LBB51_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB51_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: .LBB51_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %cmp = icmp eq i32 %b, 0
+ br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+ %a1 = add <8 x i8> %a, splat (i8 3)
+ %a2 = bitcast <8 x i8> %a1 to double
+ br label %end
+
+cmp.false:
+ %a3 = bitcast <8 x i8> %a to double
+ br label %end
+
+end:
%phi = phi double [ %a2, %cmp.true ], [ %a3, %cmp.false ]
ret double %phi
}
@@ -9009,47 +9135,45 @@ define <2 x i32> @bitcast_v4bf16_to_v2i32(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB66_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v4, v5 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: .LBB66_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -9298,46 +9422,48 @@ define inreg <2 x i32> @bitcast_v4bf16_to_v2i32_scalar(<4 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB67_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_nc_u32 v8, v8, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v5 :: v_dual_add_nc_u32 v5, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v5 :: v_dual_add_nc_u32 v9, v9, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB67_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -9785,48 +9911,56 @@ define inreg <8 x i8> @bitcast_v2i32_to_v8i8_scalar(<2 x i32> inreg %a, i32 inre
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB69_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v4, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
; GFX11-TRUE16-NEXT: s_branch .LBB69_3
; GFX11-TRUE16-NEXT: .LBB69_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: .LBB69_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s8, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s8, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s8, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB69_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_i32 s1, s1, 3
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 24
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s2
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s0
; GFX11-TRUE16-NEXT: .LBB69_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2i32_to_v8i8_scalar:
@@ -10393,62 +10527,122 @@ define inreg <2 x i32> @bitcast_v8i8_to_v2i32_scalar(<8 x i8> inreg %a, i32 inre
; GFX9-NEXT: .LBB71_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8i8_to_v2i32_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB71_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_branch .LBB71_3
-; GFX11-NEXT: .LBB71_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: .LBB71_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB71_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB71_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v2i32_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB71_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
+; GFX11-TRUE16-NEXT: s_branch .LBB71_3
+; GFX11-TRUE16-NEXT: .LBB71_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-TRUE16-NEXT: .LBB71_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: .LBB71_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v2i32_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB71_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB71_3
+; GFX11-FAKE16-NEXT: .LBB71_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-FAKE16-NEXT: .LBB71_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB71_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: .LBB71_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11781,47 +11975,45 @@ define <2 x float> @bitcast_v4bf16_to_v2f32(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB82_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v4, v5 :: v_dual_add_f32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
; GFX11-TRUE16-NEXT: .LBB82_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -12070,46 +12262,48 @@ define inreg <2 x float> @bitcast_v4bf16_to_v2f32_scalar(<4 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB83_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_nc_u32 v8, v8, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v5 :: v_dual_add_nc_u32 v5, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v2, v5 :: v_dual_add_nc_u32 v9, v9, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v5, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v9.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB83_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -12568,52 +12762,51 @@ define inreg <8 x i8> @bitcast_v2f32_to_v8i8_scalar(<2 x float> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB85_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s1
; GFX11-TRUE16-NEXT: s_branch .LBB85_3
; GFX11-TRUE16-NEXT: .LBB85_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: .LBB85_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB85_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, s0, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v9, s1, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, s0, 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB85_6
-; GFX11-TRUE16-NEXT: .LBB85_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v7, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v5, s2
-; GFX11-TRUE16-NEXT: .LBB85_6: ; %end
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-TRUE16-NEXT: .LBB85_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v2f32_to_v8i8_scalar:
@@ -13185,62 +13378,122 @@ define inreg <2 x float> @bitcast_v8i8_to_v2f32_scalar(<8 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB87_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8i8_to_v2f32_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB87_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_branch .LBB87_3
-; GFX11-NEXT: .LBB87_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: .LBB87_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB87_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB87_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v2f32_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB87_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
+; GFX11-TRUE16-NEXT: s_branch .LBB87_3
+; GFX11-TRUE16-NEXT: .LBB87_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-TRUE16-NEXT: .LBB87_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: .LBB87_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v2f32_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB87_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB87_3
+; GFX11-FAKE16-NEXT: .LBB87_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-FAKE16-NEXT: .LBB87_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB87_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: .LBB87_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -14182,50 +14435,44 @@ define <4 x i16> @bitcast_v4bf16_to_v4i16(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB94_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v6, v7 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v9 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v8, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
; GFX11-TRUE16-NEXT: .LBB94_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -14485,41 +14732,44 @@ define inreg <4 x i16> @bitcast_v4bf16_to_v4i16_scalar(<4 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB95_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s0, 16
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, 16, s0
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s1, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s0
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v7, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v6.h
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB95_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -15017,52 +15267,51 @@ define inreg <8 x i8> @bitcast_v4i16_to_v8i8_scalar(<4 x i16> inreg %a, i32 inre
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB97_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s1
; GFX11-TRUE16-NEXT: s_branch .LBB97_3
; GFX11-TRUE16-NEXT: .LBB97_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: .LBB97_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB97_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v9, s1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, s0, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB97_6
-; GFX11-TRUE16-NEXT: .LBB97_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v7, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v5, s2
-; GFX11-TRUE16-NEXT: .LBB97_6: ; %end
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-TRUE16-NEXT: .LBB97_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4i16_to_v8i8_scalar:
@@ -15655,62 +15904,122 @@ define inreg <4 x i16> @bitcast_v8i8_to_v4i16_scalar(<8 x i8> inreg %a, i32 inre
; GFX9-NEXT: .LBB99_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8i8_to_v4i16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB99_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_branch .LBB99_3
-; GFX11-NEXT: .LBB99_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: .LBB99_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB99_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB99_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4i16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB99_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
+; GFX11-TRUE16-NEXT: s_branch .LBB99_3
+; GFX11-TRUE16-NEXT: .LBB99_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-TRUE16-NEXT: .LBB99_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: .LBB99_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v4i16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB99_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB99_3
+; GFX11-FAKE16-NEXT: .LBB99_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-FAKE16-NEXT: .LBB99_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB99_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: .LBB99_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -16166,46 +16475,45 @@ define <4 x half> @bitcast_v4bf16_to_v4f16(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB102_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v0
; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v8, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v4, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: .LBB102_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -16467,48 +16775,48 @@ define inreg <4 x half> @bitcast_v4bf16_to_v4f16_scalar(<4 x bfloat> inreg %a, i
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB103_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v0, 16, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s1, 16
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v3, v7 :: v_dual_add_nc_u32 v5, v5, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v1, v3, v7 :: v_dual_add_nc_u32 v8, v8, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v8
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v2
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v5, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v3, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v4.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB103_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
@@ -17025,52 +17333,51 @@ define inreg <8 x i8> @bitcast_v4f16_to_v8i8_scalar(<4 x half> inreg %a, i32 inr
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB105_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s1
; GFX11-TRUE16-NEXT: s_branch .LBB105_3
; GFX11-TRUE16-NEXT: .LBB105_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: .LBB105_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB105_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v9, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v8, 0x200, s0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB105_6
-; GFX11-TRUE16-NEXT: .LBB105_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v7, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s5 :: v_dual_mov_b32 v5, s2
-; GFX11-TRUE16-NEXT: .LBB105_6: ; %end
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-TRUE16-NEXT: .LBB105_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4f16_to_v8i8_scalar:
@@ -17663,62 +17970,122 @@ define inreg <4 x half> @bitcast_v8i8_to_v4f16_scalar(<8 x i8> inreg %a, i32 inr
; GFX9-NEXT: .LBB107_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8i8_to_v4f16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB107_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_branch .LBB107_3
-; GFX11-NEXT: .LBB107_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: .LBB107_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB107_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB107_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4f16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB107_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
+; GFX11-TRUE16-NEXT: s_branch .LBB107_3
+; GFX11-TRUE16-NEXT: .LBB107_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-TRUE16-NEXT: .LBB107_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: .LBB107_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v4f16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB107_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB107_3
+; GFX11-FAKE16-NEXT: .LBB107_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-FAKE16-NEXT: .LBB107_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB107_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: .LBB107_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -17975,53 +18342,49 @@ define <8 x i8> @bitcast_v4bf16_to_v8i8(<4 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB108_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v2 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v2, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_lshlrev_b32 v3, 16, v8
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v7, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v9
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v9
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_add_f32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v3, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v6, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v4, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v10, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v11, vcc_lo
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v10, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
; GFX11-TRUE16-NEXT: .LBB108_4: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
@@ -18357,92 +18720,90 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB109_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[2:3], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s3 :: v_dual_mov_b32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, s1
; GFX11-TRUE16-NEXT: s_branch .LBB109_3
; GFX11-TRUE16-NEXT: .LBB109_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr7
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr2
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr5
+; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
; GFX11-TRUE16-NEXT: .LBB109_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB109_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s2, 0, s1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s1, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v1, 16, s0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s0, 0, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v2
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v7 :: v_dual_add_nc_u32 v6, v6, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v4
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 0x7fff, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x7fff, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v0, v7, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v5.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[7:8]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v8
-; GFX11-TRUE16-NEXT: s_branch .LBB109_6
-; GFX11-TRUE16-NEXT: .LBB109_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v6, s8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s5
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v1, s3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v3, s2
-; GFX11-TRUE16-NEXT: .LBB109_6: ; %end
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v9, v3, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
+; GFX11-TRUE16-NEXT: .LBB109_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v4bf16_to_v8i8_scalar:
@@ -19071,62 +19432,122 @@ define inreg <4 x bfloat> @bitcast_v8i8_to_v4bf16_scalar(<8 x i8> inreg %a, i32
; GFX9-NEXT: .LBB111_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v8i8_to_v4bf16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s20, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB111_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_perm_b32 v1, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v2, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v3, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v4, s16, s17, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX11-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX11-NEXT: s_branch .LBB111_3
-; GFX11-NEXT: .LBB111_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GFX11-NEXT: .LBB111_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB111_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v0, s18, s19, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: .LBB111_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v8i8_to_v4bf16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s19 :: v_dual_mov_b32 v3, s17
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s18 :: v_dual_mov_b32 v7, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s1 :: v_dual_mov_b32 v9, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB111_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v8, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v9, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v11, v1
+; GFX11-TRUE16-NEXT: s_branch .LBB111_3
+; GFX11-TRUE16-NEXT: .LBB111_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-TRUE16-NEXT: .LBB111_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v8
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 3, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v2, 0xc0c0004
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v3, v2
+; GFX11-TRUE16-NEXT: .LBB111_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v8i8_to_v4bf16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s20, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB111_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX11-FAKE16-NEXT: s_branch .LBB111_3
+; GFX11-FAKE16-NEXT: .LBB111_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1
+; GFX11-FAKE16-NEXT: .LBB111_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB111_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s18, s19, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: .LBB111_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index c421dcf7835b5..a3d87ac411f6e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -859,35 +859,47 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB5_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s6 :: v_dual_mov_b32 v0, s0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v11, s4
; GFX11-TRUE16-NEXT: s_branch .LBB5_3
; GFX11-TRUE16-NEXT: .LBB5_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s14, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr4_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: .LBB5_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s14, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB5_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: s_add_i32 s2, s2, 3
@@ -895,26 +907,22 @@ define inreg <12 x i8> @bitcast_v3i32_to_v12i8_scalar(<3 x i32> inreg %a, i32 in
; GFX11-TRUE16-NEXT: s_add_i32 s0, s0, 3
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v10, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s4 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v7, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v9, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, s3
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s1
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[0:1], s[2:3], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v1, s3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s0
; GFX11-TRUE16-NEXT: .LBB5_5: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, s12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, s4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, s1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, s11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, s10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, s9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, s2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, s8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, s6
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v3i32_to_v12i8_scalar:
@@ -1639,71 +1647,141 @@ define inreg <3 x i32> @bitcast_v12i8_to_v3i32_scalar(<12 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB7_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v12i8_to_v3i32_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB7_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_branch .LBB7_3
-; GFX11-NEXT: .LBB7_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX11-NEXT: .LBB7_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB7_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB7_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v3i32_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s23 :: v_dual_mov_b32 v6, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s21 :: v_dual_mov_b32 v9, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s19 :: v_dual_mov_b32 v12, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s18 :: v_dual_mov_b32 v8, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s17 :: v_dual_mov_b32 v10, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s2 :: v_dual_mov_b32 v14, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB7_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v13, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v11, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v6, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v14, v10, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v12, v7, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v9, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v15
+; GFX11-TRUE16-NEXT: s_branch .LBB7_3
+; GFX11-TRUE16-NEXT: .LBB7_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX11-TRUE16-NEXT: .LBB7_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v14
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 3, v6
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v10, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v7, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v11, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v9, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: .LBB7_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v3i32_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB7_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-FAKE16-NEXT: s_branch .LBB7_3
+; GFX11-FAKE16-NEXT: .LBB7_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX11-FAKE16-NEXT: .LBB7_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB7_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: .LBB7_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -2179,63 +2257,62 @@ define <3 x i32> @bitcast_v6bf16_to_v3i32(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB10_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v8, v6 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_cndmask_b32 v5, v7, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
; GFX11-TRUE16-NEXT: .LBB10_2: ; %end
@@ -4251,64 +4328,65 @@ define inreg <12 x i8> @bitcast_v3f32_to_v12i8_scalar(<3 x float> inreg %a, i32
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB21_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s1 :: v_dual_mov_b32 v11, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v13, s0
; GFX11-TRUE16-NEXT: s_branch .LBB21_3
; GFX11-TRUE16-NEXT: .LBB21_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: .LBB21_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB21_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
; GFX11-TRUE16-NEXT: v_add_f32_e64 v8, s2, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, s1, 1.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, s0, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v14, s1, 1.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v13, s0, 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB21_6
-; GFX11-TRUE16-NEXT: .LBB21_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v13, s13
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v7, s10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, s9 :: v_dual_mov_b32 v10, s3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, s4
-; GFX11-TRUE16-NEXT: .LBB21_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-TRUE16-NEXT: .LBB21_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v3f32_to_v12i8_scalar:
@@ -5039,71 +5117,141 @@ define inreg <3 x float> @bitcast_v12i8_to_v3f32_scalar(<12 x i8> inreg %a, i32
; GFX9-NEXT: .LBB23_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v12i8_to_v3f32_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB23_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_branch .LBB23_3
-; GFX11-NEXT: .LBB23_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
-; GFX11-NEXT: .LBB23_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB23_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB23_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v3f32_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s23 :: v_dual_mov_b32 v6, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s21 :: v_dual_mov_b32 v9, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s19 :: v_dual_mov_b32 v12, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s18 :: v_dual_mov_b32 v8, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, s17 :: v_dual_mov_b32 v10, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, s2 :: v_dual_mov_b32 v14, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB23_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v13, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v11, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v6, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v14, v10, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v12, v7, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v9, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v15
+; GFX11-TRUE16-NEXT: s_branch .LBB23_3
+; GFX11-TRUE16-NEXT: .LBB23_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX11-TRUE16-NEXT: .LBB23_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v14
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v11, 3, v11
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 3, v9
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, 3, v6
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v10, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v7, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v11, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v9, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v6, v3, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v4, v3
+; GFX11-TRUE16-NEXT: .LBB23_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v3f32_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB23_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-FAKE16-NEXT: s_branch .LBB23_3
+; GFX11-FAKE16-NEXT: .LBB23_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2
+; GFX11-FAKE16-NEXT: .LBB23_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB23_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: .LBB23_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -5590,63 +5738,62 @@ define <3 x float> @bitcast_v6bf16_to_v3f32(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB26_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v2
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v6, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_lshlrev_b32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v8, v6 :: v_dual_add_f32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v9, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_cndmask_b32 v5, v7, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v11, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v5.l
; GFX11-TRUE16-NEXT: .LBB26_2: ; %end
@@ -7889,99 +8036,169 @@ define inreg <6 x bfloat> @bitcast_v12i8_to_v6bf16_scalar(<12 x i8> inreg %a, i3
; GFX9-NEXT: .LBB37_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v12i8_to_v6bf16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB37_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_branch .LBB37_3
-; GFX11-NEXT: .LBB37_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB37_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB37_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB37_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
- %cmp = icmp eq i32 %b, 0
- br i1 %cmp, label %cmp.true, label %cmp.false
-
-cmp.true:
- %a1 = add <12 x i8> %a, splat (i8 3)
- %a2 = bitcast <12 x i8> %a1 to <6 x bfloat>
- br label %end
-
-cmp.false:
- %a3 = bitcast <12 x i8> %a to <6 x bfloat>
- br label %end
-
-end:
- %phi = phi <6 x bfloat> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
- ret <6 x bfloat> %phi
-}
-
-define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
-; SI-LABEL: bitcast_v6bf16_to_v12i8:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
+; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6bf16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s23 :: v_dual_mov_b32 v7, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s21 :: v_dual_mov_b32 v10, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s19 :: v_dual_mov_b32 v13, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v9, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v11, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s2 :: v_dual_mov_b32 v15, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB37_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v12, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v15, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v13, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB37_3
+; GFX11-TRUE16-NEXT: .LBB37_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB37_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v15
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v14
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-TRUE16-NEXT: .LBB37_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v6bf16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB37_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-FAKE16-NEXT: s_branch .LBB37_3
+; GFX11-FAKE16-NEXT: .LBB37_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB37_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB37_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: .LBB37_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+ %cmp = icmp eq i32 %b, 0
+ br i1 %cmp, label %cmp.true, label %cmp.false
+
+cmp.true:
+ %a1 = add <12 x i8> %a, splat (i8 3)
+ %a2 = bitcast <12 x i8> %a1 to <6 x bfloat>
+ br label %end
+
+cmp.false:
+ %a3 = bitcast <12 x i8> %a to <6 x bfloat>
+ br label %end
+
+end:
+ %phi = phi <6 x bfloat> [ %a2, %cmp.true ], [ %a3, %cmp.false ]
+ ret <6 x bfloat> %phi
+}
+
+define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
+; SI-LABEL: bitcast_v6bf16_to_v12i8:
+; SI: ; %bb.0:
+; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-NEXT: v_and_b32_e32 v4, 0xffff0000, v2
+; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-NEXT: v_and_b32_e32 v6, 0xffff0000, v0
+; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
; SI-NEXT: v_mul_f32_e32 v16, 1.0, v6
; SI-NEXT: v_mul_f32_e32 v17, 1.0, v0
; SI-NEXT: v_mul_f32_e32 v14, 1.0, v5
@@ -8297,65 +8514,64 @@ define <12 x i8> @bitcast_v6bf16_to_v12i8(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB38_4
; GFX11-TRUE16-NEXT: ; %bb.3: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v14.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v14
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v15.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v13.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v0, 0x40c00000, v0 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v16, 0x7fc07fc0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, 0x7fc07fc0 :: v_dual_lshlrev_b32 v3, 16, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v2, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v14
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v2, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v8, v7, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v9, v10 :: v_dual_lshlrev_b32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v4 :: v_dual_add_f32 v4, 0x40c00000, v5
-; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v10, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_add3_u32 v1, v7, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v15
+; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_cndmask_b32 v4, v7, v8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
+; GFX11-TRUE16-NEXT: v_add3_u32 v0, v0, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
@@ -8821,27 +9037,39 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB39_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s8, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v15, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s1 :: v_dual_mov_b32 v11, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v13, s0
; GFX11-TRUE16-NEXT: s_branch .LBB39_3
; GFX11-TRUE16-NEXT: .LBB39_2:
; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr8
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr15_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: .LBB39_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
@@ -8853,87 +9081,77 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s3
; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, 0x40c00000, s1
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s0
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v4, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s3
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v0
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s0
-; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s3, 0, s2
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v8, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v1
+; GFX11-TRUE16-NEXT: s_pack_lh_b32_b16 s1, 0, s2
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 16
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v12, 0x7fc07fc0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v8
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v6, v6, v3
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v5, 0x40c00000, s0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v7, 0x40c00000, s1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v5, vcc_lo
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, 0x7fff, v6
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v2, 0x40c00000, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v3, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v0, v7, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, v0, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, v9, v3
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x7fff, v0
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v9, 0x7fff, v9
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, 0x40c00000, s2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v5, 16, 1
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v4, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v8, v2 :: v_dual_add_nc_u32 v1, v3, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v9, v7
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v6.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v9, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v15
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[11:12]
-; GFX11-TRUE16-NEXT: s_branch .LBB39_6
-; GFX11-TRUE16-NEXT: .LBB39_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v5, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v9, s10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s11
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s13 :: v_dual_mov_b32 v7, s8
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s12 :: v_dual_mov_b32 v13, s9
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v11, s6
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s4
-; GFX11-TRUE16-NEXT: .LBB39_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v5.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v13.l
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v4
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x7fff, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v6.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, 0x7fc07fc0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 16, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v10, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v10.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 8, v8
+; GFX11-TRUE16-NEXT: .LBB39_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v15.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6bf16_to_v12i8_scalar:
@@ -9760,71 +9978,141 @@ define inreg <6 x half> @bitcast_v12i8_to_v6f16_scalar(<12 x i8> inreg %a, i32 i
; GFX9-NEXT: .LBB41_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v12i8_to_v6f16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB41_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_branch .LBB41_3
-; GFX11-NEXT: .LBB41_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB41_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB41_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB41_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6f16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s23 :: v_dual_mov_b32 v7, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s21 :: v_dual_mov_b32 v10, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s19 :: v_dual_mov_b32 v13, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v9, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v11, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s2 :: v_dual_mov_b32 v15, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB41_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v12, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v15, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v13, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB41_3
+; GFX11-TRUE16-NEXT: .LBB41_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB41_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v15
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v14
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-TRUE16-NEXT: .LBB41_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v6f16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB41_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-FAKE16-NEXT: s_branch .LBB41_3
+; GFX11-FAKE16-NEXT: .LBB41_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB41_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB41_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: .LBB41_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -10407,65 +10695,67 @@ define inreg <12 x i8> @bitcast_v6f16_to_v12i8_scalar(<6 x half> inreg %a, i32 i
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB43_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v13, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v15, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, s4 :: v_dual_mov_b32 v14, s0
; GFX11-TRUE16-NEXT: s_branch .LBB43_3
; GFX11-TRUE16-NEXT: .LBB43_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr14_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: .LBB43_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB43_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v4, 0x200, s1 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_pk_add_f16 v3, 0x200, s0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v15, 0x200, s1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_f16 v14, 0x200, s0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_f16 v8, 0x200, s2 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v9, 0x7e007e00
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB43_6
-; GFX11-TRUE16-NEXT: .LBB43_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v7, s10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s13 :: v_dual_mov_b32 v13, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v11, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v5, s4
-; GFX11-TRUE16-NEXT: .LBB43_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v14.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v15
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
+; GFX11-TRUE16-NEXT: .LBB43_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v14.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v15.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v13.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -11237,71 +11527,141 @@ define inreg <6 x i16> @bitcast_v12i8_to_v6i16_scalar(<12 x i8> inreg %a, i32 in
; GFX9-NEXT: .LBB45_5: ; %end
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: bitcast_v12i8_to_v6i16_scalar:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s24, 0
-; GFX11-NEXT: s_mov_b32 s4, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB45_2
-; GFX11-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v4, s22, s23, v0
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v5, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v6, s20, s21, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: v_or_b32_e32 v1, v5, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v2, v6, v3
-; GFX11-NEXT: s_branch .LBB45_3
-; GFX11-NEXT: .LBB45_2:
-; GFX11-NEXT: s_mov_b32 s4, -1
-; GFX11-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
-; GFX11-NEXT: .LBB45_3: ; %Flow
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s4, s4, exec_lo
-; GFX11-NEXT: s_cselect_b32 s4, 1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s4, 1
-; GFX11-NEXT: s_cbranch_scc1 .LBB45_5
-; GFX11-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0004
-; GFX11-NEXT: s_add_i32 s0, s0, 3
-; GFX11-NEXT: s_add_i32 s2, s2, 3
-; GFX11-NEXT: s_add_i32 s16, s16, 3
-; GFX11-NEXT: s_add_i32 s18, s18, 3
-; GFX11-NEXT: s_add_i32 s20, s20, 3
-; GFX11-NEXT: s_add_i32 s22, s22, 3
-; GFX11-NEXT: v_perm_b32 v1, s0, s1, v0
-; GFX11-NEXT: v_perm_b32 v2, s2, s3, v0
-; GFX11-NEXT: v_perm_b32 v3, s16, s17, v0
-; GFX11-NEXT: v_perm_b32 v4, s18, s19, v0
-; GFX11-NEXT: v_perm_b32 v5, s20, s21, v0
-; GFX11-NEXT: v_perm_b32 v0, s22, s23, v0
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
-; GFX11-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
-; GFX11-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
-; GFX11-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
-; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-NEXT: v_lshlrev_b32_e32 v6, 16, v0
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v4
-; GFX11-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX11-NEXT: .LBB45_5: ; %end
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: bitcast_v12i8_to_v6i16_scalar:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s23 :: v_dual_mov_b32 v7, s22
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s21 :: v_dual_mov_b32 v10, s20
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s19 :: v_dual_mov_b32 v13, s16
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v9, s3
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s17 :: v_dual_mov_b32 v11, s1
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s2 :: v_dual_mov_b32 v15, s0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
+; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB45_2
+; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v14, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v12, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v15, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v16, v13, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v17, v3
+; GFX11-TRUE16-NEXT: s_branch .LBB45_3
+; GFX11-TRUE16-NEXT: .LBB45_2:
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-TRUE16-NEXT: .LBB45_3: ; %Flow
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s0, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s0, 1
+; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 3, v15
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 3, v14
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 3, v13
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 3, v12
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 3, v10
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 3, v7
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v11, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v1, v9, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v8, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v3, v6, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v10, v5, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v7, v4, 0xc0c0004
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v4
+; GFX11-TRUE16-NEXT: .LBB45_5: ; %end
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: bitcast_v12i8_to_v6i16_scalar:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s24, 0
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
+; GFX11-FAKE16-NEXT: s_cbranch_scc0 .LBB45_2
+; GFX11-FAKE16-NEXT: ; %bb.1: ; %cmp.false
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v6, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v5, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v6, v3
+; GFX11-FAKE16-NEXT: s_branch .LBB45_3
+; GFX11-FAKE16-NEXT: .LBB45_2:
+; GFX11-FAKE16-NEXT: s_mov_b32 s4, -1
+; GFX11-FAKE16-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GFX11-FAKE16-NEXT: .LBB45_3: ; %Flow
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_and_b32 s4, s4, exec_lo
+; GFX11-FAKE16-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s4, 1
+; GFX11-FAKE16-NEXT: s_cbranch_scc1 .LBB45_5
+; GFX11-FAKE16-NEXT: ; %bb.4: ; %cmp.true
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0xc0c0004
+; GFX11-FAKE16-NEXT: s_add_i32 s0, s0, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s2, s2, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s16, s16, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s18, s18, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s20, s20, 3
+; GFX11-FAKE16-NEXT: s_add_i32 s22, s22, 3
+; GFX11-FAKE16-NEXT: v_perm_b32 v1, s0, s1, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v2, s2, s3, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v3, s16, s17, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v4, s18, s19, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v5, s20, s21, v0
+; GFX11-FAKE16-NEXT: v_perm_b32 v0, s22, s23, v0
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v1, 0x300, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x300, v2
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x300, v3
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v4, 0x300, v4
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v5, 0x300, v5
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v0, 0x300, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v4
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v5, v6
+; GFX11-FAKE16-NEXT: .LBB45_5: ; %end
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%cmp = icmp eq i32 %b, 0
br i1 %cmp, label %cmp.true, label %cmp.false
@@ -11865,65 +12225,67 @@ define inreg <12 x i8> @bitcast_v6i16_to_v12i8_scalar(<6 x i16> inreg %a, i32 in
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 0
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0
; GFX11-TRUE16-NEXT: s_cbranch_scc0 .LBB47_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.false
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s9, s2, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s10, s1, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s1, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s1, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s0, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s0, 8
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[2:3], 24
-; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[0:1], 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s4 :: v_dual_mov_b32 v9, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[6:7], s[0:1], 24
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, s5 :: v_dual_mov_b32 v2, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s0, 8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, s6
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v8, s2
+; GFX11-TRUE16-NEXT: s_lshr_b64 s[4:5], s[2:3], 24
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v14, s1 :: v_dual_mov_b32 v11, s4
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v13, s0
; GFX11-TRUE16-NEXT: s_branch .LBB47_3
; GFX11-TRUE16-NEXT: .LBB47_2:
-; GFX11-TRUE16-NEXT: s_mov_b32 s8, -1
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr14
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr12
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr4
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr13
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr11
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr10
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr9
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr3
-; GFX11-TRUE16-NEXT: ; implicit-def: $sgpr6
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, -1
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr13_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr1_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr2_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr3_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr5_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr6_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr7_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr8_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr9_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr10_lo16
+; GFX11-TRUE16-NEXT: ; implicit-def: $vgpr11_lo16
; GFX11-TRUE16-NEXT: .LBB47_3: ; %Flow
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s8, exec_lo
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, 1, 0
-; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s5, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX11-TRUE16-NEXT: s_cselect_b32 s3, 1, 0
+; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s3, 1
; GFX11-TRUE16-NEXT: s_cbranch_scc1 .LBB47_5
; GFX11-TRUE16-NEXT: ; %bb.4: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v4, s1, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v14, s1, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v9, s0, 3 op_sel_hi:[1,0]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v8, s2, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: v_pk_add_u16 v3, s0, 3 op_sel_hi:[1,0]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v4
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v13, s0, 3 op_sel_hi:[1,0]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v14
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[8:9]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[13:14]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: s_branch .LBB47_6
-; GFX11-TRUE16-NEXT: .LBB47_5:
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v4, s1
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v1, s14
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s12 :: v_dual_mov_b32 v7, s10
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v12, s13 :: v_dual_mov_b32 v9, s9
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v6, s11 :: v_dual_mov_b32 v11, s6
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, s3 :: v_dual_mov_b32 v5, s4
-; GFX11-TRUE16-NEXT: .LBB47_6: ; %end
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v12.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v14
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v13
+; GFX11-TRUE16-NEXT: .LBB47_5: ; %end
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v13.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v14.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: bitcast_v6i16_to_v12i8_scalar:
@@ -12222,60 +12584,59 @@ define <6 x half> @bitcast_v6bf16_to_v6f16(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB48_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_add_f32 v0, 0x40c00000, v0
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v6, v7 :: v_dual_add_f32 v4, 0x40c00000, v4
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v4, 16, 1
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_cndmask_b32 v0, v7, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v8, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v6, v7 :: v_dual_add_f32 v5, 0x40c00000, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v8, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v8, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_add3_u32 v8, v9, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v2, 0x40c00000, v2 :: v_dual_cndmask_b32 v5, v8, v9
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v1, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v8, v9, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v2, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v2, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v7, v10, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v6, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v5.l
; GFX11-TRUE16-NEXT: .LBB48_2: ; %end
@@ -13330,53 +13691,51 @@ define <6 x i16> @bitcast_v6bf16_to_v6i16(<6 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_and_not1_saveexec_b32 s0, s0
; GFX11-TRUE16-NEXT: s_cbranch_execz .LBB52_2
; GFX11-TRUE16-NEXT: ; %bb.1: ; %cmp.true
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, 0x40c00000, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v3
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v8, v3, 0x7fff
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v5, 0x40c00000, v5 :: v_dual_add_f32 v2, 0x40c00000, v2
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, 0x40c00000, v3 :: v_dual_lshlrev_b32 v4, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, 0x40c00000, v4
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v3, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v6, v8, vcc_lo
+; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, 0x40c00000, v0
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v0, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v0, v6, v7 :: v_dual_and_b32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 0x40c00000, v1 :: v_dual_lshlrev_b32 v4, 16, v4
-; GFX11-TRUE16-NEXT: v_dual_add_f32 v4, 0x40c00000, v4 :: v_dual_lshlrev_b32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v4, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, 0x40c00000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v8, v10, v4, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, 0x40c00000, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v7, v11 :: v_dual_and_b32 v1, 0xffff0000, v1
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 0x40c00000, v1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v11, vcc_lo
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v5, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v6, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v13, 0x400000, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v5, 0x7fff
+; GFX11-TRUE16-NEXT: v_add3_u32 v6, v6, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v6, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index 04f1b409b416b..abc410cabe583 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -29493,28 +29493,51 @@ define bfloat @v_log2_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_log2_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX11-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_log_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11TRUE16-LABEL: v_log2_bf16:
+; GFX11TRUE16: ; %bb.0:
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_log_f32_e32 v0, v0
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11FAKE16-LABEL: v_log2_bf16:
+; GFX11FAKE16: ; %bb.0:
+; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
+; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
+; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_log2_bf16:
; GFX1250TRUE16: ; %bb.0:
@@ -29924,37 +29947,69 @@ define bfloat @v_exp_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_exp_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
-; GFX11-NEXT: v_rndne_f32_e32 v2, v1
-; GFX11-NEXT: v_fma_f32 v3, 0x3fb8aa3b, v0, -v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX11-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3
-; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11-NEXT: v_exp_f32_e32 v1, v1
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
-; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
-; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11TRUE16-LABEL: v_exp_bf16:
+; GFX11TRUE16: ; %bb.0:
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x3fb8aa3b, v0, -v1
+; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
+; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x32a5705f, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11FAKE16-LABEL: v_exp_bf16:
+; GFX11FAKE16: ; %bb.0:
+; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v0
+; GFX11FAKE16-NEXT: v_rndne_f32_e32 v2, v1
+; GFX11FAKE16-NEXT: v_fma_f32 v3, 0x3fb8aa3b, v0, -v1
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11FAKE16-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX11FAKE16-NEXT: v_fmamk_f32 v3, v0, 0x32a5705f, v3
+; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX11FAKE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc2ce8ed0, v0
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11FAKE16-NEXT: v_exp_f32_e32 v1, v1
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11FAKE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX11FAKE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x42b17218, v0
+; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_exp_bf16:
; GFX1250TRUE16: ; %bb.0:
@@ -30129,28 +30184,51 @@ define bfloat @v_exp2_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_exp2_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX11-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_exp_f32_e32 v0, v0
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11TRUE16-LABEL: v_exp2_bf16:
+; GFX11TRUE16: ; %bb.0:
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo
+; GFX11TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX11TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_exp_f32_e32 v0, v0
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11FAKE16-LABEL: v_exp2_bf16:
+; GFX11FAKE16: ; %bb.0:
+; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
+; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX11FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
+; GFX11FAKE16-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_exp_f32_e32 v0, v0
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11FAKE16-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_exp2_bf16:
; GFX1250TRUE16: ; %bb.0:
@@ -30334,37 +30412,69 @@ define bfloat @v_exp10_bf16(bfloat %a) #0 {
; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_exp10_bf16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
-; GFX11-NEXT: v_rndne_f32_e32 v2, v1
-; GFX11-NEXT: v_fma_f32 v3, 0x40549a78, v0, -v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_sub_f32_e32 v1, v1, v2
-; GFX11-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3
-; GFX11-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX11-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_add_f32_e32 v1, v1, v3
-; GFX11-NEXT: v_exp_f32_e32 v1, v1
-; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_ldexp_f32 v1, v1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
-; GFX11-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
-; GFX11-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
-; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11TRUE16-LABEL: v_exp10_bf16:
+; GFX11TRUE16: ; %bb.0:
+; GFX11TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; GFX11TRUE16-NEXT: v_fma_f32 v2, 0x40549a78, v0, -v1
+; GFX11TRUE16-NEXT: v_rndne_f32_e32 v3, v1
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11TRUE16-NEXT: v_sub_f32_e32 v1, v1, v3
+; GFX11TRUE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
+; GFX11TRUE16-NEXT: v_fmamk_f32 v2, v0, 0x33979a37, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_add_f32_e32 v1, v1, v2
+; GFX11TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v3
+; GFX11TRUE16-NEXT: v_exp_f32_e32 v1, v1
+; GFX11TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11TRUE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX11TRUE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11FAKE16-LABEL: v_exp10_bf16:
+; GFX11FAKE16: ; %bb.0:
+; GFX11FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_mul_f32_e32 v1, 0x40549a78, v0
+; GFX11FAKE16-NEXT: v_rndne_f32_e32 v2, v1
+; GFX11FAKE16-NEXT: v_fma_f32 v3, 0x40549a78, v0, -v1
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11FAKE16-NEXT: v_sub_f32_e32 v1, v1, v2
+; GFX11FAKE16-NEXT: v_fmamk_f32 v3, v0, 0x33979a37, v3
+; GFX11FAKE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX11FAKE16-NEXT: v_cmp_ngt_f32_e32 vcc_lo, 0xc23369f4, v0
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_add_f32_e32 v1, v1, v3
+; GFX11FAKE16-NEXT: v_exp_f32_e32 v1, v1
+; GFX11FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11FAKE16-NEXT: v_ldexp_f32 v1, v1, v2
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX11FAKE16-NEXT: v_cmp_nlt_f32_e32 vcc_lo, 0x421a209b, v0
+; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7f800000, v1, vcc_lo
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11FAKE16-NEXT: v_bfe_u32 v1, v0, 16, 1
+; GFX11FAKE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
+; GFX11FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11FAKE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
+; GFX11FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11FAKE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250TRUE16-LABEL: v_exp10_bf16:
; GFX1250TRUE16: ; %bb.0:
@@ -30943,42 +31053,23 @@ define bfloat @v_round_bf16(bfloat %a) #0 {
; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1250TRUE16-LABEL: v_round_bf16:
-; GFX1250TRUE16: ; %bb.0:
-; GFX1250TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX1250TRUE16-NEXT: v_trunc_f32_e32 v2, v1
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
-; GFX1250TRUE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
-; GFX1250TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
-; GFX1250TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250TRUE16-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250TRUE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250TRUE16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GFX1250FAKE16-LABEL: v_round_bf16:
-; GFX1250FAKE16: ; %bb.0:
-; GFX1250FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_trunc_f32_e32 v2, v1
-; GFX1250FAKE16-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
-; GFX1250FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
-; GFX1250FAKE16-NEXT: v_add_f32_e32 v0, v2, v0
-; GFX1250FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250FAKE16-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
-; GFX1250FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GFX1250-LABEL: v_round_bf16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_trunc_f32_e32 v2, v1
+; GFX1250-NEXT: v_fma_mix_f32_bf16 v0, -v2, 1.0, v0 op_sel:[0,1,0] op_sel_hi:[0,1,1]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_cmp_ge_f32_e64 s0, |v0|, 0.5
+; GFX1250-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_bfi_b32 v0, 0x7fffffff, v0, v1
+; GFX1250-NEXT: v_add_f32_e32 v0, v2, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_cvt_pk_bf16_f32 v0, v0, s0
+; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = call bfloat @llvm.round.bf16(bfloat %a)
ret bfloat %op
}
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
index ae7dde0fec38b..ef17b1dd7527b 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fadd.ll
@@ -4966,120 +4966,63 @@ define half @buffer_fat_ptr_agent_atomic_fadd_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX12-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX12-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s4, s16, -4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
+; GFX12-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: s_cbranch_execnz .LBB16_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5124,108 +5067,57 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB16_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB16_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB16_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_addk_i32 s16, 0x200
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-NEXT: s_and_b32 s4, s16, -4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB16_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_cbranch_execnz .LBB16_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -5480,118 +5372,62 @@ define bfloat @buffer_fat_ptr_agent_atomic_fadd_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX12-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX12-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s4, s16, -4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
+; GFX12-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: s_cbranch_execnz .LBB17_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -5635,106 +5471,56 @@ define void @buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB17_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB17_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v0, v0, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB17_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_addk_i32 s16, 0x200
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-NEXT: s_and_b32 s4, s16, -4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB17_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_cbranch_execnz .LBB17_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fadd_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
index 3b2853fdc5e1e..fe0cc95dc0a51 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmax.ll
@@ -4071,120 +4071,63 @@ define half @buffer_fat_ptr_agent_atomic_fmax_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s4, s16, -4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
+; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4229,108 +4172,57 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_addk_i32 s16, 0x200
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-NEXT: s_and_b32 s4, s16, -4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_max_f32_e32 v0, v0, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4587,118 +4479,62 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmax_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v0, v0, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v0, v0, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s4, s16, -4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
+; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_max_num_f32_e32 v0, v0, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4742,106 +4578,56 @@ define void @buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v0, v0, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v0, v0, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_addk_i32 s16, 0x200
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-NEXT: s_and_b32 s4, s16, -4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_max_f32_e32 v0, v0, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmax_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
index e280847800916..6ba9f61ba7faf 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointer-atomicrmw-fmin.ll
@@ -4071,120 +4071,63 @@ define half @buffer_fat_ptr_agent_atomic_fmin_ret_f16__offset__waterfall__amdgpu
; --------------------------------------------------------------------
define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX12-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX12-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s4, s16, -4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v4, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v4, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
+; GFX12-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, v2
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: s_cbranch_execnz .LBB13_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4229,108 +4172,57 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, s6, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_lshlrev_b32 v5, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v0
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB13_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v2
-; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB13_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v2
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_addk_i32 s16, 0x200
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v0
+; GFX11-NEXT: s_and_b32 s4, s16, -4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v4, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v4, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB13_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_min_f32_e32 v0, v0, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v3, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[2:3], v4, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, v2
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_cbranch_execnz .LBB13_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v2
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -4587,118 +4479,62 @@ define bfloat @buffer_fat_ptr_agent_atomic_fmin_ret_bf16__offset__amdgpu_no_fine
}
define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory(ptr addrspace(7) inreg %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX12-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v0, v0, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_addk_co_i32 s16, 0x200
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX12-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
-; GFX12-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX12-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v0, v0, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX12-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX12-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX12-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_addk_co_i32 s16, 0x200
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_b32 s4, s16, -4
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_and_b32 s4, s16, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s4, s4, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX12-NEXT: buffer_load_b32 v1, v2, s[0:3], null offen
+; GFX12-NEXT: s_not_b32 s6, s5
+; GFX12-NEXT: s_mov_b32 s5, 0
+; GFX12-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: v_min_num_f32_e32 v0, v0, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX12-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX12-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], null offen th:TH_ATOMIC_RETURN
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX12-NEXT: v_mov_b32_e32 v1, v4
+; GFX12-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: s_cbranch_execnz .LBB14_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -4742,106 +4578,56 @@ define void @buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine
; GFX942-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_lshlrev_b32 v3, 16, v0
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-TRUE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-TRUE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-TRUE16-NEXT: s_not_b32 s6, s5
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, 0
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v0, v0, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-TRUE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-TRUE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-TRUE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_addk_i32 s16, 0x200
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, -4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s16, 3
-; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s5, 0xffff, s4
-; GFX11-FAKE16-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
-; GFX11-FAKE16-NEXT: s_not_b32 s6, s5
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, 0
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB14_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, s4, v1
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v0, v0, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, s4, v0
-; GFX11-FAKE16-NEXT: v_and_or_b32 v0, v1, s6, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
-; GFX11-FAKE16-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, v4
-; GFX11-FAKE16-NEXT: s_or_b32 s5, vcc_lo, s5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB14_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: s_addk_i32 s16, 0x200
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-NEXT: s_and_b32 s4, s16, -4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: s_and_b32 s4, s16, 3
+; GFX11-NEXT: s_lshl_b32 s4, s4, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX11-NEXT: buffer_load_b32 v1, v2, s[0:3], 0 offen
+; GFX11-NEXT: s_not_b32 s6, s5
+; GFX11-NEXT: s_mov_b32 s5, 0
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB14_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, s4, v1
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: v_min_f32_e32 v0, v0, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v4, v0, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v5, 0x400000, v0
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v0, v4, v5, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX11-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
+; GFX11-NEXT: buffer_atomic_cmpswap_b32 v[4:5], v2, s[0:3], 0 offen glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v4, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, v4
+; GFX11-NEXT: s_or_b32 s5, vcc_lo, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_cbranch_execnz .LBB14_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: buffer_fat_ptr_agent_atomic_fmin_noret_bf16__offset__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index ad9f317c8842b..f487f9249bc33 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -6476,25 +6476,27 @@ define amdgpu_kernel void @test_call_external_void_func_v16i8() #0 {
; GFX11-TRUE16-NEXT: s_addc_u32 s3, s3, external_void_func_v16i8 at rel32@hi+12
; GFX11-TRUE16-NEXT: s_mov_b32 s32, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_load_b128 v[16:19], off, s[4:7], 0
+; GFX11-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[4:7], 0
; GFX11-TRUE16-NEXT: s_mov_b64 s[6:7], s[0:1]
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v18.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[2:3]
; GFX11-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
index d31e9ba894f6f..efe3e1df951c4 100644
--- a/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
+++ b/llvm/test/CodeGen/AMDGPU/calling-conventions.ll
@@ -131,29 +131,17 @@ define amdgpu_ps half @ps_ret_cc_inreg_f16(half inreg %arg0) {
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s0, 1.0
; GFX11-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-TRUE16-LABEL: ps_ret_cc_inreg_f16:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: s_add_f16 s0, s0, 1.0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1250-TRUE16-NEXT: ; return to shader part epilog
-;
-; GFX1250-FAKE16-LABEL: ps_ret_cc_inreg_f16:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-FAKE16-NEXT: v_nop
-; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: s_add_f16 s0, s0, 1.0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-LABEL: ps_ret_cc_inreg_f16:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_add_f16 s0, s0, 1.0
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
+; GFX1250-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-NEXT: ; return to shader part epilog
%add = fadd half %arg0, 1.0
ret half %add
}
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index e201f1c02e60c..1d484acac3f58 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -2839,20 +2839,19 @@ define bfloat @fmul_select_bf16_test1(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3f80
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4000, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x4000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3026,20 +3025,19 @@ define bfloat @fmul_select_bf16_test2(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3f80
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x3f00, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x3f00, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -3818,20 +3816,19 @@ define bfloat @fmul_select_bf16_test5(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4100
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4000, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x4000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4007,20 +4004,19 @@ define bfloat @fmul_select_bf16_test6(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4040
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xc100, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0xc100, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4195,20 +4191,19 @@ define bfloat @fmul_select_bf16_test7(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc080
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x4100, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x4100, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4563,20 +4558,19 @@ define bfloat @fmul_select_bf16_test9(bfloat %x, i32 %bool.arg1, i32 %bool.arg2)
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xc200
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xc180, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0xc180, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4752,20 +4746,19 @@ define bfloat @fmul_select_bf16_test10_sel_log2val_pos65_pos56(bfloat %x, i32 %b
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0xdb80
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0xe000, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0xe000, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4941,20 +4934,19 @@ define bfloat @fmul_select_bf16_test11_sel_log2val_neg22_pos25(bfloat %x, i32 %b
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x4c00
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, 0x3480, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v0.h, 0x3480, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v1, v0
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-SDAG-TRUE16-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_add3_u32 v1, v1, v0, 0x7fff
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
index 19c5213956606..a65db4605daf7 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll
@@ -75,10 +75,18 @@ define amdgpu_ps i16 @s_test_copysign_f16_0(half inreg %mag) {
; GFX9-NEXT: s_and_b32 s0, s0, 0x7fff
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_test_copysign_f16_0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s0, s0, 0x7fff
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_0:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_0:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half %mag, half 0.0)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -100,10 +108,18 @@ define amdgpu_ps i16 @s_test_copysign_f16_1(half inreg %mag) {
; GFX9-NEXT: s_and_b32 s0, s0, 0x7fff
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_test_copysign_f16_1:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s0, s0, 0x7fff
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_1:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_1:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half %mag, half 1.0)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -125,10 +141,18 @@ define amdgpu_ps i16 @s_test_copysign_f16_10.0(half inreg %mag) {
; GFX9-NEXT: s_and_b32 s0, s0, 0x7fff
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_test_copysign_f16_10.0:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s0, s0, 0x7fff
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_10.0:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_10.0:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half %mag, half 10.0)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -150,10 +174,18 @@ define amdgpu_ps i16 @s_test_copysign_f16_neg1(half inreg %mag) {
; GFX9-NEXT: s_bitset1_b32 s0, 15
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_test_copysign_f16_neg1:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_bitset1_b32 s0, 15
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_neg1:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_bitset1_b32 s0, 15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_neg1:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_bitset1_b32 s0, 15
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half %mag, half -1.0)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -175,10 +207,18 @@ define amdgpu_ps i16 @s_test_copysign_f16_neg10(half inreg %mag) {
; GFX9-NEXT: s_bitset1_b32 s0, 15
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_test_copysign_f16_neg10:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_bitset1_b32 s0, 15
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_neg10:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_bitset1_b32 s0, 15
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_neg10:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_bitset1_b32 s0, 15
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half %mag, half -10.0)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -200,10 +240,18 @@ define amdgpu_ps i16 @s_test_copysign_f16_0_mag(half inreg %sign) {
; GFX9-NEXT: s_and_b32 s0, s0, 0x8000
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_test_copysign_f16_0_mag:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_0_mag:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x8000, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_0_mag:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half 0.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -229,12 +277,22 @@ define amdgpu_ps i16 @s_test_copysign_f16_1_mag(half inreg %sign) {
; GFX9-NEXT: s_or_b32 s0, s0, 0x3c00
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_test_copysign_f16_1_mag:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, 0x3c00
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_1_mag:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x8000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x3c00, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_1_mag:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 0x3c00
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half 1.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -265,14 +323,23 @@ define i16 @s_test_copysign_f16_10_mag(half inreg %sign) {
; GFX9-NEXT: v_mov_b32_e32 v0, s4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: s_test_copysign_f16_10_mag:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, 0x4900
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_10_mag:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x8000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x4900, v0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_10_mag:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 0x4900
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call half @llvm.copysign.f16(half 10.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -297,12 +364,22 @@ define amdgpu_ps i16 @s_test_copysign_f16_neg1_mag(half inreg %sign) {
; GFX9-NEXT: s_or_b32 s0, s0, 0x3c00
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_test_copysign_f16_neg1_mag:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, 0x3c00
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_neg1_mag:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x8000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x3c00, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_neg1_mag:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 0x3c00
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half -1.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -327,12 +404,22 @@ define amdgpu_ps i16 @s_test_copysign_f16_neg10_mag(half inreg %sign) {
; GFX9-NEXT: s_or_b32 s0, s0, 0x4900
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_test_copysign_f16_neg10_mag:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s0, 0x4900
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_test_copysign_f16_neg10_mag:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x8000, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, 0x4900, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_test_copysign_f16_neg10_mag:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, 0x4900
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%result = call half @llvm.copysign.f16(half -10.0, half %sign)
%cast = bitcast half %result to i16
ret i16 %cast
@@ -6558,10 +6645,18 @@ define amdgpu_ps i32 @s_copysign_f16_0_f16(half inreg %sign) {
; GFX9-NEXT: s_and_b32 s0, s0, 0x8000
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX11-LABEL: s_copysign_f16_0_f16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_and_b32 s0, s0, 0x8000
-; GFX11-NEXT: ; return to shader part epilog
+; GFX11-TRUE16-LABEL: s_copysign_f16_0_f16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x8000, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_copysign_f16_0_f16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x8000
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%op = call half @llvm.copysign.f16(half 0.0, half %sign)
%cast = bitcast half %op to i16
%zext = zext i16 %cast to i32
@@ -6936,7 +7031,7 @@ define half @v_copysign_f16_0_f64(double %sign) {
; GFX11-TRUE16-LABEL: v_copysign_f16_0_f64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x8000, v1.h
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x8000, v1.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_copysign_f16_0_f64:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
index 35c407758254b..1c851a66486d4 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fadd.ll
@@ -11940,114 +11940,58 @@ define void @flat_system_atomic_fadd_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB46_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB46_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB46_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12092,106 +12036,54 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB46_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB46_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB46_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB46_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB46_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12391,119 +12283,61 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB47_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB47_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB47_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12550,110 +12384,56 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB47_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB47_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB47_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB47_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB47_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12859,119 +12639,61 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB48_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB48_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB48_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -13019,110 +12741,56 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB48_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB48_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB48_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB48_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB48_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14028,33 +13696,32 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -14062,7 +13729,7 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB51_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -14147,41 +13814,40 @@ define bfloat @flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB51_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB51_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -14670,111 +14336,56 @@ define void @flat_agent_atomic_fadd_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX12-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX12-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB53_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v6, v3
+; GFX12-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB53_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14818,103 +14429,52 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB53_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB53_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB53_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v6, v3
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB53_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB53_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -15109,121 +14669,62 @@ define void @flat_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -15270,110 +14771,56 @@ define bfloat @flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
index d4f94afe193a6..21c2f1b530b63 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmax.ll
@@ -9786,114 +9786,58 @@ define void @flat_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9938,106 +9882,54 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10238,119 +10130,61 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10397,110 +10231,56 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10707,119 +10487,61 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10867,110 +10589,56 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11177,111 +10845,56 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
}
define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v6, v3
+; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -11319,109 +10932,58 @@ define void @flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB39_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB39_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v6, v3
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12319,33 +11881,32 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -12353,7 +11914,7 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12438,41 +11999,40 @@ define bfloat @flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12963,121 +12523,62 @@ define void @flat_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -13124,110 +12625,56 @@ define bfloat @flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
index def9fe8c0f6b4..dd0bae5226c3a 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fmin.ll
@@ -9786,114 +9786,58 @@ define void @flat_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_gr
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9938,106 +9882,54 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10238,119 +10130,61 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(pt
}
define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10397,110 +10231,56 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10707,119 +10487,61 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_gr
}
define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10867,110 +10589,56 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11177,111 +10845,56 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_gr
}
define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v6, v3
+; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -11319,109 +10932,58 @@ define void @flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(pt
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB39_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB39_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v6, v3
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -12319,33 +11881,32 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -12353,7 +11914,7 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12438,41 +11999,40 @@ define bfloat @flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no
; GFX11-TRUE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12963,121 +12523,62 @@ define void @flat_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_no
}
define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -13124,110 +12625,56 @@ define bfloat @flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_g
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
index be2aa7d9f0a0d..cc5204d00efdd 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-atomicrmw-fsub.ll
@@ -9382,114 +9382,58 @@ define void @flat_system_atomic_fsub_noret_f16__offset12b_pos(ptr %ptr, half %va
; --------------------------------------------------------------------
define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16:
; GFX942: ; %bb.0:
@@ -9534,106 +9478,54 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16:
; GFX10: ; %bb.0:
@@ -9833,119 +9725,61 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16(ptr %ptr, bfloat %val) #0 {
}
define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -9992,110 +9826,56 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
@@ -10301,119 +10081,61 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %
}
define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX942: ; %bb.0:
@@ -10451,120 +10173,66 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
; GFX942-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] sc0
; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB34_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB34_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
+; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX10: ; %bb.0:
@@ -10770,111 +10438,56 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr %ptr, bfloat %
}
define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: flat_load_b32 v4, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v6, v3
+; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_agent_atomic_fsub_noret_bf16:
; GFX942: ; %bb.0:
@@ -10913,108 +10526,57 @@ define void @flat_agent_atomic_fsub_noret_bf16(ptr %ptr, bfloat %val) #0 {
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB35_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_noret_bf16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: flat_load_b32 v4, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_cbranch_execnz .LBB35_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: flat_agent_atomic_fsub_noret_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: flat_load_b32 v4, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v6, v3
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_agent_atomic_fsub_noret_bf16:
; GFX10: ; %bb.0:
@@ -11909,33 +11471,32 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11943,7 +11504,7 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -12028,41 +11589,40 @@ define bfloat @flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr %ptr,
; GFX11-TRUE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: flat_load_b32 v2, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: flat_load_b32 v3, v[0:1] offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v2, v[0:1], v[4:5] offset:2046 glc
+; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v3, v[0:1], v[3:4] offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: flat_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -12551,121 +12111,62 @@ define void @flat_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr %ptr,
}
define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: flat_load_b32 v5, v[0:1]
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -12712,110 +12213,56 @@ define bfloat @flat_system_atomic_fsub_ret_bf16__offset12b_pos(ptr %ptr, bfloat
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: flat_load_b32 v5, v[0:1]
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: flat_load_b32 v5, v[0:1]
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: flat_atomic_cmpswap_b32 v5, v[0:1], v[5:6] glc
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: flat_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
index 477c58bef7542..1229882e1710d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
@@ -132,18 +132,18 @@ define amdgpu_ps void @flat_store_saddr_i8_zext_vgpr_offset_neg2048(ptr inreg %s
; Base pointer is uniform, but also in VGPRs
define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs(i32 %voffset, i8 %data) {
-; GFX1250-SDAG-LABEL: flat_store_saddr_uniform_ptr_in_vgprs:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-SDAG-NEXT: v_nop
-; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v1
-; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
-; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2
-; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-FAKE16-LABEL: flat_store_saddr_uniform_ptr_in_vgprs:
+; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
+; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: ds_load_b64 v[4:5], v1
+; GFX1250-SDAG-FAKE16-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-SDAG-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs:
; GFX1250-GISEL: ; %bb.0:
@@ -159,6 +159,20 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs(i32 %voffset, i8 %d
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: flat_store_b8 v[2:3], v1
; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-REAL16-LABEL: flat_store_saddr_uniform_ptr_in_vgprs:
+; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-REAL16-NEXT: v_nop
+; GFX1250-SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-REAL16-NEXT: ds_load_b64 v[4:5], v1
+; GFX1250-SDAG-REAL16-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
+; GFX1250-SDAG-REAL16-NEXT: flat_store_b8 v[0:1], v2
+; GFX1250-SDAG-REAL16-NEXT: s_endpgm
%sbase = load ptr, ptr addrspace(3) @ptr.in.lds
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -168,18 +182,18 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs(i32 %voffset, i8 %d
; Base pointer is uniform, but also in VGPRs, with imm offset
define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voffset, i8 %data) {
-; GFX1250-SDAG-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-SDAG-NEXT: v_nop
-; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
-; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v1
-; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
-; GFX1250-SDAG-NEXT: flat_store_b8 v[0:1], v2 offset:-120
-; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-FAKE16-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset:
+; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
+; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: ds_load_b64 v[4:5], v1
+; GFX1250-SDAG-FAKE16-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_store_b8 v[0:1], v2 offset:-120
+; GFX1250-SDAG-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-GISEL: ; %bb.0:
@@ -195,6 +209,20 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voff
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: flat_store_b8 v[2:3], v1 offset:-120
; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-REAL16-LABEL: flat_store_saddr_uniform_ptr_in_vgprs_immoffset:
+; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-REAL16-NEXT: v_nop
+; GFX1250-SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-REAL16-NEXT: ds_load_b64 v[4:5], v1
+; GFX1250-SDAG-REAL16-NEXT: s_wait_dscnt 0x0
+; GFX1250-SDAG-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], v[4:5], v[0:1]
+; GFX1250-SDAG-REAL16-NEXT: flat_store_b8 v[0:1], v2 offset:-120
+; GFX1250-SDAG-REAL16-NEXT: s_endpgm
%sbase = load ptr, ptr addrspace(3) @ptr.in.lds
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
@@ -208,17 +236,17 @@ define amdgpu_ps void @flat_store_saddr_uniform_ptr_in_vgprs_immoffset(i32 %voff
; --------------------------------------------------------------------------------
define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr(ptr inreg %sbase, i32 %voffset, i16 %data) {
-; GFX1250-SDAG-LABEL: flat_store_saddr_i16_zext_vgpr:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-SDAG-NEXT: v_nop
-; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_store_b16 v[0:1], v2
-; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-FAKE16-LABEL: flat_store_saddr_i16_zext_vgpr:
+; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
+; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_store_b16 v[0:1], v2
+; GFX1250-SDAG-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_i16_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
@@ -232,6 +260,19 @@ define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: flat_store_b16 v[2:3], v1
; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-REAL16-LABEL: flat_store_saddr_i16_zext_vgpr:
+; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-REAL16-NEXT: v_nop
+; GFX1250-SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-REAL16-NEXT: flat_store_b16 v[0:1], v2
+; GFX1250-SDAG-REAL16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
store i16 %data, ptr %gep0
@@ -239,17 +280,17 @@ define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, i16 %data) {
-; GFX1250-SDAG-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-SDAG-NEXT: v_nop
-; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_store_b16 v[0:1], v2 offset:-128
-; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-FAKE16-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128:
+; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
+; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_store_b16 v[0:1], v2 offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
@@ -263,6 +304,19 @@ define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr_offset_neg128(ptr inreg %s
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: flat_store_b16 v[2:3], v1 offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-REAL16-LABEL: flat_store_saddr_i16_zext_vgpr_offset_neg128:
+; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-REAL16-NEXT: v_nop
+; GFX1250-SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-REAL16-NEXT: flat_store_b16 v[0:1], v2 offset:-128
+; GFX1250-SDAG-REAL16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -271,17 +325,17 @@ define amdgpu_ps void @flat_store_saddr_i16_zext_vgpr_offset_neg128(ptr inreg %s
}
define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr(ptr inreg %sbase, i32 %voffset, half %data) {
-; GFX1250-SDAG-LABEL: flat_store_saddr_f16_zext_vgpr:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-SDAG-NEXT: v_nop
-; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_store_b16 v[0:1], v2
-; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-FAKE16-LABEL: flat_store_saddr_f16_zext_vgpr:
+; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
+; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_store_b16 v[0:1], v2
+; GFX1250-SDAG-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_f16_zext_vgpr:
; GFX1250-GISEL: ; %bb.0:
@@ -295,6 +349,19 @@ define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr(ptr inreg %sbase, i32 %vof
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: flat_store_b16 v[2:3], v1
; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-REAL16-LABEL: flat_store_saddr_f16_zext_vgpr:
+; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-REAL16-NEXT: v_nop
+; GFX1250-SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-REAL16-NEXT: flat_store_b16 v[0:1], v2
+; GFX1250-SDAG-REAL16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
store half %data, ptr %gep0
@@ -302,17 +369,17 @@ define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr(ptr inreg %sbase, i32 %vof
}
define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, half %data) {
-; GFX1250-SDAG-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128:
-; GFX1250-SDAG: ; %bb.0:
-; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-SDAG-NEXT: v_nop
-; GFX1250-SDAG-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
-; GFX1250-SDAG-NEXT: flat_store_b16 v[0:1], v2 offset:-128
-; GFX1250-SDAG-NEXT: s_endpgm
+; GFX1250-SDAG-FAKE16-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128:
+; GFX1250-SDAG-FAKE16: ; %bb.0:
+; GFX1250-SDAG-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-FAKE16-NEXT: v_nop
+; GFX1250-SDAG-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-FAKE16-NEXT: v_dual_mov_b32 v2, v1 :: v_dual_mov_b32 v1, 0
+; GFX1250-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-FAKE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-FAKE16-NEXT: flat_store_b16 v[0:1], v2 offset:-128
+; GFX1250-SDAG-FAKE16-NEXT: s_endpgm
;
; GFX1250-GISEL-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128:
; GFX1250-GISEL: ; %bb.0:
@@ -326,6 +393,19 @@ define amdgpu_ps void @flat_store_saddr_f16_zext_vgpr_offset_neg128(ptr inreg %s
; GFX1250-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1250-GISEL-NEXT: flat_store_b16 v[2:3], v1 offset:-128
; GFX1250-GISEL-NEXT: s_endpgm
+;
+; GFX1250-SDAG-REAL16-LABEL: flat_store_saddr_f16_zext_vgpr_offset_neg128:
+; GFX1250-SDAG-REAL16: ; %bb.0:
+; GFX1250-SDAG-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-SDAG-REAL16-NEXT: v_nop
+; GFX1250-SDAG-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b16_e32 v2.l, v1.l
+; GFX1250-SDAG-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-SDAG-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-REAL16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-SDAG-REAL16-NEXT: flat_store_b16 v[0:1], v2 offset:-128
+; GFX1250-SDAG-REAL16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -2124,5 +2204,3 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg12
; GFX1250: {{.*}}
; GFX1250-GISEL-FAKE16: {{.*}}
; GFX1250-GISEL-REAL16: {{.*}}
-; GFX1250-SDAG-FAKE16: {{.*}}
-; GFX1250-SDAG-REAL16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
index bad68376c242e..a3e559edaa694 100644
--- a/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
+++ b/llvm/test/CodeGen/AMDGPU/float-to-arbitrary-fp-fp8-hw.ll
@@ -1596,40 +1596,38 @@ define i8 @to_fp8_bf16(bfloat %x) {
; GFX1200-NEXT: s_wait_samplecnt 0x0
; GFX1200-NEXT: s_wait_bvhcnt 0x0
; GFX1200-NEXT: s_wait_kmcnt 0x0
-; GFX1200-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v0
; GFX1200-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_lshlrev_b32_e32 v5, 16, v1
-; GFX1200-NEXT: v_and_b16 v0.l, 0x80, v0.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_frexp_mant_f32_e32 v1, v5
; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v5
+; GFX1200-NEXT: v_and_b16 v0.l, 0x80, v0.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_xor_b16 v0.h, v6.l, -1
; GFX1200-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX1200-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_add3_u32 v2, v2, v1, 0x7fff
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_min_u16 v0.h, v0.h, 15
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc_lo
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc_lo
; GFX1200-NEXT: v_sub_nc_u16 v1.l, v0.h, 1 clamp
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1200-NEXT: v_lshrrev_b32_e32 v3, 16, v7
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_lshlrev_b16 v1.h, v1.l, 1
; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 19, v7
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_and_b16 v2.l, 0x7f, v3.l
; GFX1200-NEXT: v_and_b16 v2.h, v3.l, 7
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_add_nc_u16 v1.h, v1.h, -1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX1200-NEXT: v_or_b16 v3.l, 0x80, v2.l
; GFX1200-NEXT: v_lshrrev_b16 v2.l, 4, v2.l
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_and_b16 v1.h, v3.l, v1.h
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX1200-NEXT: v_and_b16 v2.h, v2.l, 1
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc_lo
@@ -1690,74 +1688,73 @@ define i8 @to_fp8_bf16(bfloat %x) {
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
+; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v0
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.h, 0x80
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v1
-; GFX1250-TRUE16-NEXT: v_and_b16 v0.l, 0x80, v0.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v7, v6
; GFX1250-TRUE16-NEXT: v_frexp_mant_f32_e32 v1, v6
+; GFX1250-TRUE16-NEXT: v_and_b16 v0.l, 0x80, v0.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_xor_b16 v0.h, v7.l, -1
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_cvt_pk_bf16_f32 v1, v1, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_min_u16 v0.h, v0.h, 15
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, 0x7f, v1.l
; GFX1250-TRUE16-NEXT: v_and_b16 v2.h, v1.l, 7
-; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.h, v0.h, 1 clamp
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-TRUE16-NEXT: v_sub_nc_u16 v1.h, v0.h, 1 clamp
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v4.l, 4, v2.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.h
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v3.l, v1.h, 1
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_add_nc_u16 v2.h, v3.l, -1
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v3.l, v1.l, v3.h, 0x7f bitop3:0xec
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.l, 3, v1.l
; GFX1250-TRUE16-NEXT: v_and_b16 v3.h, v4.l, 1
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v2.l, v2.l, v2.h, 0x80 bitop3:0xc8
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v2.h, v0.h, v3.l
; GFX1250-TRUE16-NEXT: v_lshrrev_b16 v1.h, v1.h, v3.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v1.l, v5.l, v3.h bitop3:0xe0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v2.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_and_b16 v2.l, v2.h, 1
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.l, v4.l, v1.l
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_ne_u16_e32 vcc_lo, 0, v0.h
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e64 s0, 7, v1.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, 0, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v1.h, v3.l, v2.l bitop3:0xe0
; GFX1250-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v6
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v2.l, v4.l
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, 0, v0.h, vcc_lo
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v7.l, v2.l
-; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.h, v0.h
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-TRUE16-NEXT: v_add_nc_u16 v0.h, v2.h, v0.h
; GFX1250-TRUE16-NEXT: v_add_nc_u16 v1.h, v1.h, 6
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_cmp_lt_i16_e32 vcc_lo, 7, v0.h
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX1250-TRUE16-NEXT: v_lshlrev_b16 v2.l, 3, v1.h
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v0.h, 0, vcc_lo
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v2.h, 0, 8, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_gt_i16_e32 vcc_lo, 1, v1.h
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v1.l, v0.l, v1.l, v2.l bitop3:0xfe
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_bitop3_b16 v0.h, v0.l, v0.h, v2.h bitop3:0xfe
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.h, v1.l, v0.h, vcc_lo
; GFX1250-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v6, v6
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s0
+; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7f, v0.l, vcc_lo
; GFX1250-TRUE16-NEXT: s_set_pc_i64 s[30:31]
;
@@ -2625,178 +2622,183 @@ define <4 x i8> @to_e5m3fnu_v4f32(<4 x float> %x) {
; GFX1200-NEXT: s_wait_kmcnt 0x0
; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v4, v1
; GFX1200-NEXT: v_frexp_mant_f32_e32 v5, v1
-; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v6, v0
+; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v7, v0
; GFX1200-NEXT: s_mov_b32 s1, 0x7fffff
-; GFX1200-NEXT: v_frexp_mant_f32_e32 v7, v0
+; GFX1200-NEXT: v_frexp_mant_f32_e32 v6, v0
; GFX1200-NEXT: v_sub_nc_u32_e32 v8, 7, v4
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-NEXT: v_and_or_b32 v13, v5, s1, 0x800000
-; GFX1200-NEXT: v_sub_nc_u32_e32 v11, 7, v6
-; GFX1200-NEXT: v_and_b32_e32 v14, 0x7ffff, v5
-; GFX1200-NEXT: v_and_or_b32 v17, v7, s1, 0x800000
+; GFX1200-NEXT: v_and_or_b32 v12, v5, s1, 0x800000
+; GFX1200-NEXT: v_and_b32_e32 v13, 0x7ffff, v5
+; GFX1200-NEXT: v_sub_nc_u32_e32 v15, 7, v7
+; GFX1200-NEXT: v_and_or_b32 v17, v6, s1, 0x800000
; GFX1200-NEXT: v_min_u32_e32 v8, 31, v8
-; GFX1200-NEXT: v_bfe_u32 v15, v5, 20, 3
-; GFX1200-NEXT: v_min_u32_e32 v11, 31, v11
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v14
+; GFX1200-NEXT: v_bfe_u32 v14, v5, 20, 3
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v13
+; GFX1200-NEXT: v_min_u32_e32 v15, 31, v15
; GFX1200-NEXT: v_lshrrev_b32_e32 v5, 19, v5
; GFX1200-NEXT: v_sub_nc_u32_e64 v16, v8, 1 clamp
-; GFX1200-NEXT: v_lshrrev_b32_e32 v22, v8, v13
-; GFX1200-NEXT: v_sub_nc_u32_e64 v18, v11, 1 clamp
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
-; GFX1200-NEXT: v_lshrrev_b32_e32 v23, v11, v17
-; GFX1200-NEXT: v_bfe_u32 v20, v13, 0, v16
-; GFX1200-NEXT: v_lshrrev_b32_e32 v13, v16, v13
-; GFX1200-NEXT: v_bfe_u32 v21, v17, 0, v18
-; GFX1200-NEXT: v_and_or_b32 v14, v15, 1, v14
-; GFX1200-NEXT: v_lshrrev_b32_e32 v17, v18, v17
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX1200-NEXT: v_lshrrev_b32_e32 v22, v8, v12
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc_lo
+; GFX1200-NEXT: v_sub_nc_u32_e64 v21, v15, 1 clamp
+; GFX1200-NEXT: v_cmp_ne_u32_e64 s0, 0, v15
+; GFX1200-NEXT: v_bfe_u32 v20, v12, 0, v16
+; GFX1200-NEXT: v_lshrrev_b32_e32 v12, v16, v12
+; GFX1200-NEXT: v_and_or_b32 v13, v14, 1, v13
+; GFX1200-NEXT: v_bfe_u32 v24, v17, 0, v21
; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v10, v3
-; GFX1200-NEXT: v_frexp_mant_f32_e32 v9, v3
-; GFX1200-NEXT: v_and_b32_e32 v5, v5, v14
-; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v12, v2
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v20
+; GFX1200-NEXT: v_and_b32_e32 v18, 0x7ffff, v6
+; GFX1200-NEXT: v_and_b32_e32 v5, v5, v13
+; GFX1200-NEXT: v_lshrrev_b32_e32 v13, v21, v17
+; GFX1200-NEXT: v_sub_nc_u32_e32 v23, 7, v10
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v20, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v21
-; GFX1200-NEXT: v_sub_nc_u32_e32 v14, 7, v10
-; GFX1200-NEXT: v_add_nc_u32_e32 v5, v15, v5
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v24
+; GFX1200-NEXT: v_add_nc_u32_e32 v5, v14, v5
+; GFX1200-NEXT: v_frexp_mant_f32_e32 v9, v3
+; GFX1200-NEXT: v_min_u32_e32 v23, 31, v23
; GFX1200-NEXT: v_and_or_b32 v16, v22, 1, v20
+; GFX1200-NEXT: v_lshrrev_b32_e32 v20, v15, v17
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v24, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
-; GFX1200-NEXT: v_min_u32_e32 v14, 31, v14
-; GFX1200-NEXT: v_and_or_b32 v20, v9, s1, 0x800000
-; GFX1200-NEXT: v_and_b32_e32 v13, v13, v16
-; GFX1200-NEXT: v_and_b32_e32 v19, 0x7ffff, v7
-; GFX1200-NEXT: v_and_or_b32 v18, v23, 1, v21
+; GFX1200-NEXT: v_sub_nc_u32_e64 v14, v23, 1 clamp
+; GFX1200-NEXT: v_and_b32_e32 v12, v12, v16
+; GFX1200-NEXT: v_frexp_exp_i32_f32_e32 v11, v2
+; GFX1200-NEXT: v_and_or_b32 v16, v20, 1, v24
+; GFX1200-NEXT: v_bfe_u32 v19, v6, 20, 3
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_cndmask_b32_e32 v8, 0, v13, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v11
-; GFX1200-NEXT: v_and_b32_e32 v16, v17, v18
-; GFX1200-NEXT: v_sub_nc_u32_e32 v13, 7, v12
-; GFX1200-NEXT: v_lshrrev_b32_e32 v18, v14, v20
-; GFX1200-NEXT: v_add_nc_u32_e32 v8, v22, v8
-; GFX1200-NEXT: v_bfe_u32 v22, v9, 20, 3
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v11, 0, v16, vcc_lo
+; GFX1200-NEXT: v_dual_cndmask_b32 v8, 0, v12 :: v_dual_and_b32 v21, 0x7ffff, v9
; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v5
-; GFX1200-NEXT: v_sub_nc_u32_e64 v16, v14, 1 clamp
-; GFX1200-NEXT: v_cmp_lt_i32_e64 s0, 7, v8
-; GFX1200-NEXT: v_min_u32_e32 v13, 31, v13
+; GFX1200-NEXT: v_and_b32_e32 v13, v13, v16
+; GFX1200-NEXT: v_and_or_b32 v12, v9, s1, 0x800000
+; GFX1200-NEXT: v_lshrrev_b32_e32 v6, 19, v6
+; GFX1200-NEXT: v_add_nc_u32_e32 v8, v22, v8
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc_lo
+; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, 0, v13, s0
; GFX1200-NEXT: v_add_co_ci_u32_e64 v4, null, 14, v4, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i32_e64 s0, 7, v8
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v18
+; GFX1200-NEXT: v_frexp_mant_f32_e32 v18, v2
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX1200-NEXT: v_lshl_or_b32 v5, v4, 3, v5
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v8, v8, 0, s0
; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 8, s0
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_lshl_or_b32 v5, v4, 3, v5
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b32_e32 v22, 0x7ffff, v18
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX1200-NEXT: v_or_b32_e32 v8, v15, v8
-; GFX1200-NEXT: v_add_nc_u32_e32 v11, v23, v11
-; GFX1200-NEXT: v_bfe_u32 v15, v20, 0, v16
-; GFX1200-NEXT: v_lshrrev_b32_e32 v16, v16, v20
-; GFX1200-NEXT: v_sub_nc_u32_e64 v20, v13, 1 clamp
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_4) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v11
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v11, v11, 0, vcc_lo
+; GFX1200-NEXT: v_add_nc_u32_e32 v13, v20, v13
+; GFX1200-NEXT: v_bfe_u32 v15, v12, 0, v14
+; GFX1200-NEXT: v_and_or_b32 v16, v19, 1, v16
+; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v13
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-NEXT: v_and_b32_e32 v6, v6, v16
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e64 v13, v13, 0, vcc_lo
; GFX1200-NEXT: v_cndmask_b32_e64 v17, 0, 8, vcc_lo
; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v4
-; GFX1200-NEXT: v_or_b32_e32 v11, v17, v11
+; GFX1200-NEXT: v_add_nc_u32_e32 v6, v19, v6
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_or_b32_e32 v13, v17, v13
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v4, v5, v8, vcc_lo
+; GFX1200-NEXT: v_sub_nc_u32_e32 v8, 7, v11
; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v15
-; GFX1200-NEXT: v_frexp_mant_f32_e32 v15, v2
-; GFX1200-NEXT: v_and_b32_e32 v21, 0x7ffff, v9
-; GFX1200-NEXT: v_bfe_u32 v17, v7, 20, 3
-; GFX1200-NEXT: v_lshrrev_b32_e32 v7, 19, v7
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v19
-; GFX1200-NEXT: v_and_or_b32 v19, v15, s1, 0x800000
-; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 19, v9
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1200-NEXT: v_and_or_b32 v5, v18, 1, v5
+; GFX1200-NEXT: v_lshrrev_b32_e32 v5, v23, v12
+; GFX1200-NEXT: v_lshrrev_b32_e32 v12, v14, v12
+; GFX1200-NEXT: v_and_or_b32 v14, v18, s1, 0x800000
+; GFX1200-NEXT: v_min_u32_e32 v8, 31, v8
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc_lo
; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v21
-; GFX1200-NEXT: v_lshrrev_b32_e32 v23, v13, v19
-; GFX1200-NEXT: v_and_b32_e32 v5, v16, v5
-; GFX1200-NEXT: v_bfe_u32 v16, v19, 0, v20
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v21, 0, 1, vcc_lo
-; GFX1200-NEXT: v_and_or_b32 v8, v17, 1, v8
-; GFX1200-NEXT: v_lshrrev_b32_e32 v19, v20, v19
-; GFX1200-NEXT: v_bfe_u32 v20, v15, 20, 3
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v16
-; GFX1200-NEXT: v_and_or_b32 v21, v22, 1, v21
-; GFX1200-NEXT: v_and_b32_e32 v7, v7, v8
+; GFX1200-NEXT: v_bfe_u32 v17, v9, 20, 3
+; GFX1200-NEXT: v_lshrrev_b32_e32 v9, 19, v9
+; GFX1200-NEXT: v_sub_nc_u32_e64 v20, v8, 1 clamp
+; GFX1200-NEXT: v_and_or_b32 v15, v5, 1, v15
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX1200-NEXT: v_and_b32_e32 v14, 0x7ffff, v15
-; GFX1200-NEXT: v_and_b32_e32 v9, v9, v21
-; GFX1200-NEXT: v_lshrrev_b32_e32 v15, 19, v15
-; GFX1200-NEXT: v_and_or_b32 v16, v23, 1, v16
+; GFX1200-NEXT: v_lshrrev_b32_e32 v21, v8, v14
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX1200-NEXT: v_and_b32_e32 v12, v12, v15
+; GFX1200-NEXT: v_bfe_u32 v15, v14, 0, v20
+; GFX1200-NEXT: v_and_or_b32 v16, v17, 1, v16
+; GFX1200-NEXT: v_lshrrev_b32_e32 v14, v20, v14
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v15
+; GFX1200-NEXT: v_and_b32_e32 v9, v9, v16
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v14
-; GFX1200-NEXT: v_add_nc_u32_e32 v9, v22, v9
-; GFX1200-NEXT: v_add_nc_u32_e32 v7, v17, v7
-; GFX1200-NEXT: v_and_b32_e32 v8, v19, v16
-; GFX1200-NEXT: v_add_nc_u32_e32 v5, v18, v5
+; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v23
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_add_nc_u32_e32 v9, v17, v9
+; GFX1200-NEXT: v_and_or_b32 v15, v21, 1, v15
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc_lo
-; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v13
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_and_or_b32 v14, v20, 1, v14
+; GFX1200-NEXT: v_cndmask_b32_e32 v12, 0, v12, vcc_lo
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v22
+; GFX1200-NEXT: v_bfe_u32 v22, v18, 20, 3
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1200-NEXT: v_add_nc_u32_e32 v5, v5, v12
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v20, 0, 1, vcc_lo
+; GFX1200-NEXT: v_and_b32_e32 v12, v14, v15
+; GFX1200-NEXT: v_lshrrev_b32_e32 v15, 19, v18
+; GFX1200-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_and_or_b32 v14, v22, 1, v20
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v8, 0, v12, vcc_lo
; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v5
-; GFX1200-NEXT: v_and_b32_e32 v13, v15, v14
+; GFX1200-NEXT: v_and_b32_e32 v12, v15, v14
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_add_nc_u32_e32 v8, v23, v8
+; GFX1200-NEXT: v_add_nc_u32_e32 v8, v21, v8
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v5, v5, 0, vcc_lo
; GFX1200-NEXT: v_cndmask_b32_e64 v14, 0, 8, vcc_lo
; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v9
-; GFX1200-NEXT: v_add_nc_u32_e32 v13, v20, v13
+; GFX1200-NEXT: v_add_nc_u32_e32 v12, v22, v12
; GFX1200-NEXT: v_cmp_lt_i32_e64 s0, 7, v8
; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX1200-NEXT: v_or_b32_e32 v5, v14, v5
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v9, v9, 0, vcc_lo
; GFX1200-NEXT: v_add_co_ci_u32_e64 v10, null, 14, v10, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v13
+; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v12
; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX1200-NEXT: v_cndmask_b32_e64 v8, v8, 0, s0
; GFX1200-NEXT: v_cndmask_b32_e64 v15, 0, 8, s0
; GFX1200-NEXT: v_lshl_or_b32 v9, v10, 3, v9
-; GFX1200-NEXT: v_cmp_gt_i32_e64 s0, 1, v10
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v13, v13, 0, vcc_lo
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v12, null, 14, v12, vcc_lo
-; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v7
+; GFX1200-NEXT: v_cndmask_b32_e64 v12, v12, 0, vcc_lo
+; GFX1200-NEXT: v_add_co_ci_u32_e64 v11, null, 14, v11, vcc_lo
+; GFX1200-NEXT: v_cmp_lt_i32_e32 vcc_lo, 7, v6
; GFX1200-NEXT: v_or_b32_e32 v8, v15, v8
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v5, v9, v5, s0
-; GFX1200-NEXT: v_lshl_or_b32 v13, v12, 3, v13
-; GFX1200-NEXT: v_cmp_gt_i32_e64 s0, 1, v12
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX1200-NEXT: v_lshl_or_b32 v12, v11, 3, v12
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v7, v7, 0, vcc_lo
-; GFX1200-NEXT: v_add_co_ci_u32_e64 v6, null, 14, v6, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc_lo
+; GFX1200-NEXT: v_add_co_ci_u32_e64 v7, null, 14, v7, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v10
+; GFX1200-NEXT: v_lshl_or_b32 v6, v7, 3, v6
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v5, v9, v5, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v11
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v8, v12, v8, vcc_lo
; GFX1200-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v3
-; GFX1200-NEXT: s_wait_alu depctr_va_sdst(0)
-; GFX1200-NEXT: v_cndmask_b32_e64 v8, v13, v8, s0
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_lshl_or_b32 v7, v6, 3, v7
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v5, 0, v5, vcc_lo
; GFX1200-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v2
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v8, 0, v8, vcc_lo
+; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v7
+; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX1200-NEXT: v_cndmask_b32_e32 v6, v6, v13, vcc_lo
; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v3, v3
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v3, 0xff, v5, vcc_lo
@@ -2805,35 +2807,30 @@ define <4 x i8> @to_e5m3fnu_v4f32(<4 x float> %x) {
; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v5, 0xff, v8, vcc_lo
-; GFX1200-NEXT: v_cmp_gt_i32_e32 vcc_lo, 1, v6
-; GFX1200-NEXT: v_and_b16 v2.h, 0xff, v5.l
-; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v6, v7, v11, vcc_lo
; GFX1200-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v1
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX1200-NEXT: v_or_b16 v2.h, v2.h, v2.l
+; GFX1200-NEXT: v_and_b16 v2.h, 0xff, v5.l
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_or_b16 v2.l, v2.h, v2.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v4, 0, v4, vcc_lo
; GFX1200-NEXT: v_cmp_neq_f32_e32 vcc_lo, 0, v0
+; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc_lo
+; GFX1200-NEXT: v_cndmask_b32_e32 v6, 0, v6, vcc_lo
; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-NEXT: v_cndmask_b32_e32 v1, 0xff, v4, vcc_lo
; GFX1200-NEXT: v_cmp_o_f32_e32 vcc_lo, v0, v0
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_3)
-; GFX1200-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
+; GFX1200-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX1200-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
; GFX1200-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-NEXT: v_cndmask_b32_e32 v0, 0xff, v3, vcc_lo
-; GFX1200-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1200-NEXT: v_cndmask_b32_e32 v0, 0xff, v6, vcc_lo
+; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX1200-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX1200-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX1200-NEXT: v_lshrrev_b32_e32 v3, 24, v3
-; GFX1200-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX1200-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX1200-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX1200-NEXT: s_setpc_b64 s[30:31]
;
; GFX1250-TRUE16-LABEL: to_e5m3fnu_v4f32:
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
index b7f72c58c5264..a6a0cdd4241e9 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3-maximumnum.ll
@@ -1909,11 +1909,8 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_max3_bf16_maximumnum_maximumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
@@ -1927,30 +1924,29 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2004,11 +2000,8 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
@@ -2027,29 +2020,28 @@ define bfloat @v_max3_bf16_maximumnum_maximumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
index 785cb88c1a379..0b1b5a6359547 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3.bf16.ll
@@ -69,17 +69,16 @@ define bfloat @v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum(bfloat %a) #1 {
; GFX11-SDAG-TRUE16-LABEL: v_test_fmed3_r_i_i_bf16_minimumnum_maximumnum:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_o_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, 2.0, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4000, v0.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 4.0, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x4080, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
index 7676c58005d90..18178a9cc138a 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3-minimumnum.ll
@@ -1911,11 +1911,8 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-LABEL: v_min3_bf16_minimumnum_minimumnum__v_v_v_0:
; GFX11-SDAG-TRUE16: ; %bb.0:
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
@@ -1929,30 +1926,29 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
-; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX11-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2006,11 +2002,8 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v4, v4
@@ -2029,29 +2022,28 @@ define bfloat @v_min3_bf16_minimumnum_minimumnum__v_v_v_0(bfloat %a, bfloat %b,
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v3
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
-; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.l
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v1, v1
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX12-SDAG-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.l, v2.l, vcc_lo
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v2.l, v0.l, s0
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v2, v3
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-SDAG-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
; GFX12-SDAG-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-SDAG-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
diff --git a/llvm/test/CodeGen/AMDGPU/fpow.ll b/llvm/test/CodeGen/AMDGPU/fpow.ll
index 780feeaa80725..bcb109c1f6a59 100644
--- a/llvm/test/CodeGen/AMDGPU/fpow.ll
+++ b/llvm/test/CodeGen/AMDGPU/fpow.ll
@@ -800,47 +800,46 @@ define half @v_pow_f16(half %x, half %y) {
; GFX11-TRUE16-LABEL: v_pow_f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v2.l
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x800000, |v3|
-; GFX11-TRUE16-NEXT: v_cmp_class_f32_e64 s2, v3, 24
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 0x42000000, s0
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_trunc_f32_e32 v7, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x800000, |v2|
+; GFX11-TRUE16-NEXT: v_cmp_class_f32_e64 s2, v2, 24
+; GFX11-TRUE16-NEXT: v_cmp_neq_f32_e64 s1, v7, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 32, s0
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42000000, s0
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0.5
-; GFX11-TRUE16-NEXT: v_ldexp_f32 v2, |v3|, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_log_f32_e32 v5, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
; GFX11-TRUE16-NEXT: v_fma_mix_f32 v1, v1, s0, neg(0) op_sel_hi:[1,0,0]
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_ldexp_f32 v4, |v2|, v4
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, v7, v5
; GFX11-TRUE16-NEXT: v_trunc_f32_e32 v6, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_log_f32_e32 v4, v4
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v4, v5, v4
-; GFX11-TRUE16-NEXT: v_trunc_f32_e32 v7, v2
-; GFX11-TRUE16-NEXT: v_mul_dx9_zero_f32_e32 v4, v2, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, v7, v2
-; GFX11-TRUE16-NEXT: v_cmp_neq_f32_e64 s1, v7, v2
-; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 0x42800000, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v4, v4, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v5, 0, 0xffffffc0, vcc_lo
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v4, v3
+; GFX11-TRUE16-NEXT: v_mul_dx9_zero_f32_e32 v3, v5, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v3
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 0x42800000, vcc_lo
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 0xffffffc0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_lg_f32_e32 vcc_lo, v6, v1
-; GFX11-TRUE16-NEXT: v_exp_f32_e32 v4, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_exp_f32_e32 v3, v3
; GFX11-TRUE16-NEXT: s_and_b32 vcc_lo, s0, vcc_lo
; GFX11-TRUE16-NEXT: s_and_b32 s0, s2, s1
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-TRUE16-NEXT: v_ldexp_f32 v4, v4, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v4, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc_lo
+; GFX11-TRUE16-NEXT: v_ldexp_f32 v3, v3, v4
+; GFX11-TRUE16-NEXT: v_bfi_b32 v0, 0x7fffffff, v3, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc_lo
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, v0, 0x7fc00000, s0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
index eeaf84d142b06..66b2ba892182a 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-scalar.ll
@@ -561,53 +561,21 @@ define i16 @test_s_signed_i16_f32(float inreg %f) nounwind {
; GFX11-NEXT: v_cvt_pk_i16_f32 v0, s0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-FAKE16-LABEL: test_s_signed_i16_f32:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_cvt_i32_f32 s0, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_min_i32 s0, s0, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_max_i32 s0, s0, 0xffff8000
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-TRUE16-LABEL: test_s_signed_i16_f32:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_cvt_i32_f32 s0, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_min_i32 s0, s0, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_max_i32 s0, s0, 0xffff8000
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-GI-LABEL: test_s_signed_i16_f32:
-; GFX12-GI: ; %bb.0:
-; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GI-NEXT: s_wait_expcnt 0x0
-; GFX12-GI-NEXT: s_wait_samplecnt 0x0
-; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: s_cvt_i32_f32 s0, s0
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: s_min_i32 s0, s0, 0x7fff
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffff8000
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: test_s_signed_i16_f32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_cvt_i32_f32 s0, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_min_i32 s0, s0, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_max_i32 s0, s0, 0xffff8000
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%x = call i16 @llvm.fptosi.sat.i16.f32(float %f)
ret i16 %x
}
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index a746a1e97511e..f58a4d748cd36 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -1961,7 +1961,8 @@ define <2 x i1> @test_s_signed_v2f64_v2i1(<2 x double> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s0, s0, -1
; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i1> @llvm.fptosi.sat.v2f64.v2i1(<2 x double> %f)
ret <2 x i1> %x
@@ -2058,7 +2059,8 @@ define <2 x i8> @test_s_signed_v2f64_v2i8(<2 x double> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s0, s0, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <2 x i8> @llvm.fptosi.sat.v2f64.v2i8(<2 x double> %f)
ret <2 x i8> %x
@@ -2560,13 +2562,15 @@ define <4 x i1> @test_signed_v4f16_v4i1(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v4f16_v4i1:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, -1, 0
-; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v0.h, -1, 0
-; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, -1, 0
+; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v0.h, -1, 0
+; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, -1, 0
; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v1.h, -1, 0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2596,13 +2600,15 @@ define <4 x i1> @test_signed_v4f16_v4i1(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v3.l
; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, -1, 0
-; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v0.h, -1, 0
-; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, -1, 0
+; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v0.h, -1, 0
+; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, -1, 0
; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v1.h, -1, 0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2734,25 +2740,26 @@ define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v4f16_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xff80
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0xff80
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, v2.l, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v2.l, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v2.l, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v2.l, 0x7f
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
+; GFX11-TRUE16-NEXT: v_med3_i16 v0.h, v1.l, v1.h, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v1.h, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v2.l, v1.h, 0x7f
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_med3_i16 v1.h, v2.l, v1.h, 0x7f
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.h
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2795,25 +2802,26 @@ define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, 0xff80
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, 0xff80
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, v2.l, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v2.l, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v2.l, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v2.l, 0x7f
-; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_med3_i16 v0.h, v1.l, v1.h, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v1.h, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v2.l, v1.h, 0x7f
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v3.l
+; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX12-TRUE16-NEXT: v_med3_i16 v1.h, v2.l, v1.h, 0x7f
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.h
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -2923,10 +2931,12 @@ define <4 x i16> @test_signed_v4f16_v4i16(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v4f16_v4i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v4f16_v4i16:
@@ -2953,10 +2963,12 @@ define <4 x i16> @test_signed_v4f16_v4i16(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v4f16_v4i16:
@@ -3037,14 +3049,16 @@ define <4 x i32> @test_signed_v4f16_v4i32_duplicate(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v4f16_v4i32_duplicate:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v2
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v3
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v4
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v1
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v4
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v4f16_v4i32_duplicate:
@@ -3073,14 +3087,16 @@ define <4 x i32> @test_signed_v4f16_v4i32_duplicate(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v2
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v3
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v4
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v1
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v4
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v4f16_v4i32_duplicate:
@@ -3181,14 +3197,16 @@ define <4 x i64> @test_signed_v4f16_v4i64(<4 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v4f16_v4i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v2
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v3
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v5
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v1
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v3
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
@@ -3225,14 +3243,16 @@ define <4 x i64> @test_signed_v4f16_v4i64(<4 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v2
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v3
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v5
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v1
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v3
; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
@@ -3457,8 +3477,10 @@ define <4 x i1> @test_s_signed_v4f16_v4i1(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s1, s1, -1
; GFX12-GI-NEXT: s_max_i32 s3, s3, -1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i1> @llvm.fptosi.sat.v4f16.v4i1(<4 x half> %f)
ret <4 x i1> %x
@@ -3725,8 +3747,10 @@ define <4 x i8> @test_s_signed_v4f16_v4i8(<4 x half> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s1, s1, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s2
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptosi.sat.v4f16.v4i8(<4 x half> %f)
ret <4 x i8> %x
@@ -4188,19 +4212,24 @@ define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v8f16_v8i1:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
-; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, -1, 0
-; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v1.l, -1, 0
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v4.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v2.l, -1, 0
+; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.h, -1, 0
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v6.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v7.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v8.l
; GFX11-TRUE16-NEXT: v_med3_i16 v4.l, v4.l, -1, 0
-; GFX11-TRUE16-NEXT: v_med3_i16 v6.l, v3.l, -1, 0
-; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v0.h, -1, 0
+; GFX11-TRUE16-NEXT: v_med3_i16 v6.l, v0.h, -1, 0
+; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, -1, 0
; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v1.h, -1, 0
; GFX11-TRUE16-NEXT: v_med3_i16 v5.l, v2.h, -1, 0
; GFX11-TRUE16-NEXT: v_med3_i16 v7.l, v3.h, -1, 0
@@ -4243,19 +4272,24 @@ define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v4.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
-; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, -1, 0
-; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v1.l, -1, 0
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v4.l, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
+; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v2.l, -1, 0
+; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.h, -1, 0
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v6.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v7.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v5.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v8.l
; GFX12-TRUE16-NEXT: v_med3_i16 v4.l, v4.l, -1, 0
-; GFX12-TRUE16-NEXT: v_med3_i16 v6.l, v3.l, -1, 0
-; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v0.h, -1, 0
+; GFX12-TRUE16-NEXT: v_med3_i16 v6.l, v0.h, -1, 0
+; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, -1, 0
; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v1.h, -1, 0
; GFX12-TRUE16-NEXT: v_med3_i16 v5.l, v2.h, -1, 0
; GFX12-TRUE16-NEXT: v_med3_i16 v7.l, v3.h, -1, 0
@@ -4268,23 +4302,23 @@ define <8 x i1> @test_signed_v8f16_v8i1(<8 x half> %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_mov_b32_e32 v4, v2
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.l, v1.l
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, v1.l
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.h, v4.h
; GFX12-GI-NEXT: v_med3_i16 v1.l, v0.h, -1, 0
-; GFX12-GI-NEXT: v_med3_i16 v8.l, v4.l, -1, 0
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v2.l
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, v2.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.h, v3.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v4.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.h, v3.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
; GFX12-GI-NEXT: v_med3_i16 v0.l, v0.l, -1, 0
+; GFX12-GI-NEXT: v_med3_i16 v2.l, v2.l, -1, 0
; GFX12-GI-NEXT: v_med3_i16 v3.l, v1.h, -1, 0
; GFX12-GI-NEXT: v_med3_i16 v4.l, v0.h, -1, 0
-; GFX12-GI-NEXT: v_med3_i16 v5.l, v2.l, -1, 0
-; GFX12-GI-NEXT: v_med3_i16 v6.l, v2.h, -1, 0
+; GFX12-GI-NEXT: v_med3_i16 v5.l, v2.h, -1, 0
+; GFX12-GI-NEXT: v_med3_i16 v6.l, v4.h, -1, 0
; GFX12-GI-NEXT: v_med3_i16 v7.l, v3.h, -1, 0
-; GFX12-GI-NEXT: v_mov_b32_e32 v2, v8
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -4479,45 +4513,49 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v8f16_v8i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0xff80
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i16 v3.h, v3.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v1.l
+; GFX11-TRUE16-NEXT: v_med3_i16 v1.h, v3.h, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v3.l, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v6.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v5.l
; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_med3_i16 v2.l, v2.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX11-TRUE16-NEXT: v_med3_i16 v2.h, v2.h, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i16 v0.h, v3.h, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v3.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.h
+; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v5.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v0.h
; GFX11-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v4.l, 0x7f
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v2.h
+; GFX11-TRUE16-NEXT: v_med3_i16 v1.l, v3.h, v4.l, 0x7f
+; GFX11-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v8f16_v8i8:
@@ -4581,45 +4619,49 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, 0xff80
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i16 v3.h, v3.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v5.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v1.l
+; GFX12-TRUE16-NEXT: v_med3_i16 v1.h, v3.h, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v3.l, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v6.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v5.l
; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.l, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v1.l, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i16 v1.h, v1.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_and_b16 v3.l, 0xff, v3.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v3.h
-; GFX12-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v2.l
-; GFX12-TRUE16-NEXT: v_med3_i16 v2.l, v2.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX12-TRUE16-NEXT: v_med3_i16 v2.h, v2.h, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i16 v3.l, v3.l, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i16 v0.h, v3.h, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v2.l
; GFX12-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX12-TRUE16-NEXT: v_or_b16 v6.h, v3.l, v3.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.h, 8, v2.h
+; GFX12-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v5.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
+; GFX12-TRUE16-NEXT: v_or_b16 v4.h, v1.l, v2.h
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v2.l, v0.h
; GFX12-TRUE16-NEXT: v_med3_i16 v0.l, v0.l, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i16 v0.h, v0.h, v4.l, 0x7f
-; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v1.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v6.h
-; GFX12-TRUE16-NEXT: v_or_b16 v6.l, v4.h, v2.h
+; GFX12-TRUE16-NEXT: v_med3_i16 v1.l, v3.h, v4.l, 0x7f
+; GFX12-TRUE16-NEXT: v_or_b16 v4.l, v1.h, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v0.h
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v3.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v6
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v6.h
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 16, v5
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[6:7], 24, v[3:4]
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v4
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v6.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.h
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v8f16_v8i8:
@@ -4629,24 +4671,24 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
; GFX12-GI-NEXT: s_wait_samplecnt 0x0
; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: v_mov_b16_e32 v4.h, 0xff80
+; GFX12-GI-NEXT: v_mov_b32_e32 v4, v2
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.h, 0xff80
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.l, v1.l
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
-; GFX12-GI-NEXT: v_med3_i16 v1.l, v0.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v8.l, v4.l, v4.h, 0x7f
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, v1.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v4.h, v4.h
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v5.h, v3.l
+; GFX12-GI-NEXT: v_med3_i16 v1.l, v0.h, v2.h, 0x7f
; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v0.h, v1.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, v2.l
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.l, v2.h
-; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v2.h, v3.l
-; GFX12-GI-NEXT: v_med3_i16 v0.l, v0.l, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v3.l, v0.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v4.l, v1.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v5.l, v2.l, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v6.l, v2.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_med3_i16 v7.l, v3.h, v4.h, 0x7f
-; GFX12-GI-NEXT: v_mov_b32_e32 v2, v8
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v1.h, v4.l
+; GFX12-GI-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX12-GI-NEXT: v_med3_i16 v0.l, v0.l, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v2.l, v2.l, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v3.l, v0.h, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v4.l, v1.h, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v5.l, v4.h, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v6.l, v5.h, v2.h, 0x7f
+; GFX12-GI-NEXT: v_med3_i16 v7.l, v3.h, v2.h, 0x7f
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptosi.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -4793,14 +4835,18 @@ define <8 x i16> @test_signed_v8f16_v8i16(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v8f16_v8i16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v6.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v7.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v8f16_v8i16:
@@ -4835,14 +4881,18 @@ define <8 x i16> @test_signed_v8f16_v8i16(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v3.h
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.h, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.h, v5.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.h, v6.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v0.h, v7.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v1.l, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v2.l, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v3.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v8f16_v8i16:
@@ -4973,24 +5023,28 @@ define <8 x i32> @test_signed_v8f16_v8i32_duplicate(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v8f16_v8i32_duplicate:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v9, v4
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v8, v5
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v3.h
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v6
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v7
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v0
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v5
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v5, v2
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v7, v10
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v9
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v8
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v6.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v7.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v8, v4
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v9, v5
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v2.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v5, v6
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v7, v7
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v1
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v4
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v3
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v8
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v9
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v8f16_v8i32_duplicate:
@@ -5031,24 +5085,28 @@ define <8 x i32> @test_signed_v8f16_v8i32_duplicate(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v9, v4
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v8, v5
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v3.h
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v6
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v7
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v0
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v3, v5
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v5, v2
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v7, v10
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v9
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v8
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v5.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v6.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v7.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v8, v4
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v9, v5
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v2.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v5, v6
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v7, v7
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v1
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v4
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v3
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v8
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v9
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v8f16_v8i32_duplicate:
@@ -5218,31 +5276,35 @@ define <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
; GFX11-TRUE16-LABEL: test_signed_v8f16_v8i64:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v3.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.h
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v8, v6
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v1.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v2.h
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v8, v6.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.l
; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v16, v4
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v5
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v12, v7
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v0
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v10, v10
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v14, v3
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v16
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v7.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v5
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v10, v8
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v14, v4
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v1
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v8, v2
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v12, v3
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v16
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; GFX11-TRUE16-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, v16 :: v_dual_mov_b32 v3, v17
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_signed_v8f16_v8i64:
@@ -5290,31 +5352,35 @@ define <8 x i64> @test_signed_v8f16_v8i64(<8 x half> %f) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v2.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v0.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v1.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v3.l
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.h
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v8, v6
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v1.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v2.h
-; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.h
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v0.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v4.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v5, v5.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v8, v6.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v1, v1.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.l
; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v16, v4
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v5
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v12, v7
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v0
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v10, v10
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v14, v3
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v16
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v4, v7.l
+; GFX12-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v6, v5
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v10, v8
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v14, v4
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v1
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v8, v2
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v12, v3
; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, v16
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, v16 :: v_dual_mov_b32 v3, v17
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_signed_v8f16_v8i64:
@@ -5695,11 +5761,15 @@ define <8 x i1> @test_s_signed_v8f16_v8i1(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GI-NEXT: s_max_i32 s6, s6, -1
; GFX12-GI-NEXT: s_max_i32 s7, s7, -1
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s5
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s4
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s5
+; GFX12-GI-NEXT: v_mov_b16_e32 v4.l, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v5.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-GI-NEXT: v_mov_b16_e32 v6.l, s6
+; GFX12-GI-NEXT: v_mov_b16_e32 v7.l, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i1> @llvm.fptosi.sat.v8f16.v8i1(<8 x half> %f)
ret <8 x i1> %x
@@ -5952,29 +6022,29 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, s2, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i32 v3, v3, s2, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i32 v6, v4, s2, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i32 v5, v5, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i32 v4, v4, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v6.l, s0
; GFX11-TRUE16-NEXT: v_med3_i32 v9, v2, s2, 0x7f
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v1, v0, 0x6050400
-; GFX11-TRUE16-NEXT: v_cvt_i16_f16_e32 v8.l, s0
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v6, v3, 0x6050400
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v5, v7, 0x6050400
+; GFX11-TRUE16-NEXT: v_med3_i32 v5, v5, s2, 0x7f
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v1, v0, 0x6050400
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v4, v3, 0x6050400
+; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v6, 0, 16
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v9
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v4
-; GFX11-TRUE16-NEXT: v_bfe_i32 v0, v8, 0, 16
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xff00, v1
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v5, v7, 0x6050400
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v8
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v3, v11
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xff00, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[9:10], 24, v[7:8]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v5, v4
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v3, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_signed_v8f16_v8i8:
@@ -6068,29 +6138,29 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i32 v1, v1, s2, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i32 v3, v3, s2, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i32 v6, v4, s2, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i32 v5, v5, s2, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i32 v4, v4, s2, 0x7f
+; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v6.l, s0
; GFX12-TRUE16-NEXT: v_med3_i32 v9, v2, s2, 0x7f
-; GFX12-TRUE16-NEXT: v_perm_b32 v4, v1, v0, 0x6050400
-; GFX12-TRUE16-NEXT: v_cvt_i16_f16_e32 v8.l, s0
-; GFX12-TRUE16-NEXT: v_perm_b32 v2, v6, v3, 0x6050400
-; GFX12-TRUE16-NEXT: v_perm_b32 v6, v5, v7, 0x6050400
+; GFX12-TRUE16-NEXT: v_med3_i32 v5, v5, s2, 0x7f
+; GFX12-TRUE16-NEXT: v_perm_b32 v8, v1, v0, 0x6050400
+; GFX12-TRUE16-NEXT: v_perm_b32 v2, v4, v3, 0x6050400
+; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v6, 0, 16
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v9
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v4
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v8, 0, 16
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xff00, v1
+; GFX12-TRUE16-NEXT: v_perm_b32 v6, v5, v7, 0x6050400
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v8
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, s2, 0x7f
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, v3, v11
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v5, v7
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xff00, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[9:10], 24, v[7:8]
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, v5, v4
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v3, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_signed_v8f16_v8i8:
@@ -6165,11 +6235,15 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GI-NEXT: s_max_i32 s6, s6, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s7, s7, 0xffffff80
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s5
-; GFX12-GI-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s4
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s5
+; GFX12-GI-NEXT: v_mov_b16_e32 v4.l, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v5.l, s1
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
+; GFX12-GI-NEXT: v_mov_b16_e32 v6.l, s6
+; GFX12-GI-NEXT: v_mov_b16_e32 v7.l, s7
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <8 x i8> @llvm.fptosi.sat.v8f16.v8i8(<8 x half> %f)
ret <8 x i8> %x
@@ -7015,21 +7089,20 @@ define <4 x i8> @test_signed_v4f32_v4i8(<4 x float> %f) {
; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX11-TRUE16-NEXT: s_movk_i32 s0, 0xff80
; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX11-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v0
; GFX11-TRUE16-NEXT: v_med3_i32 v3, v3, s0, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i32 v4, v2, s0, 0x7f
+; GFX11-TRUE16-NEXT: v_med3_i32 v2, v2, s0, 0x7f
; GFX11-TRUE16-NEXT: v_med3_i32 v1, v1, s0, 0x7f
-; GFX11-TRUE16-NEXT: v_med3_i32 v0, v0, s0, 0x7f
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v3.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
+; GFX11-TRUE16-NEXT: v_med3_i32 v3, v4, s0, 0x7f
+; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v0.l
+; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -7074,22 +7147,21 @@ define <4 x i8> @test_signed_v4f32_v4i8(<4 x float> %f) {
; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v2, v2
; GFX12-TRUE16-NEXT: s_movk_i32 s0, 0xff80
; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX12-TRUE16-NEXT: v_cvt_i32_f32_e32 v4, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_med3_i32 v3, v3, s0, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i32 v4, v2, s0, 0x7f
+; GFX12-TRUE16-NEXT: v_med3_i32 v2, v2, s0, 0x7f
; GFX12-TRUE16-NEXT: v_med3_i32 v1, v1, s0, 0x7f
-; GFX12-TRUE16-NEXT: v_med3_i32 v0, v0, s0, 0x7f
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v3.l
-; GFX12-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v4.l
-; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX12-TRUE16-NEXT: v_or_b16 v2.h, v2.h, v2.l
-; GFX12-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v2.l
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v3.l
+; GFX12-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v2.l
+; GFX12-TRUE16-NEXT: v_med3_i32 v3, v4, s0, 0x7f
+; GFX12-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
+; GFX12-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v0.l
+; GFX12-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v3.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v3
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -7264,8 +7336,10 @@ define <4 x i8> @test_s_signed_v4f32_v4i8(<4 x float> inreg %f) {
; GFX12-GI-NEXT: s_max_i32 s2, s2, 0xffffff80
; GFX12-GI-NEXT: s_max_i32 s3, s3, 0xffffff80
; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-GI-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX12-GI-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX12-GI-NEXT: v_mov_b16_e32 v1.l, s1
+; GFX12-GI-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-GI-NEXT: v_mov_b16_e32 v3.l, s3
; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i8> @llvm.fptosi.sat.v4f32.v4i8(<4 x float> %f)
ret <4 x i8> %x
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
index a2e4f3bc150fb..8b6df1ca1e69c 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-scalar.ll
@@ -407,47 +407,19 @@ define i16 @test_s_unsigned_i16_f32(float inreg %f) nounwind {
; GFX11-NEXT: v_cvt_pk_u16_f32 v0, s0, s0
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-FAKE16-LABEL: test_s_unsigned_i16_f32:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_min_u32 s0, s0, 0xffff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-TRUE16-LABEL: test_s_unsigned_i16_f32:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_min_u32 s0, s0, 0xffff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-GI-LABEL: test_s_unsigned_i16_f32:
-; GFX12-GI: ; %bb.0:
-; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-GI-NEXT: s_wait_expcnt 0x0
-; GFX12-GI-NEXT: s_wait_samplecnt 0x0
-; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
-; GFX12-GI-NEXT: s_wait_kmcnt 0x0
-; GFX12-GI-NEXT: s_cvt_u32_f32 s0, s0
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: s_min_u32 s0, s0, 0xffff
-; GFX12-GI-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-GI-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: test_s_unsigned_i16_f32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_cvt_u32_f32 s0, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_min_u32 s0, s0, 0xffff
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%x = call i16 @llvm.fptoui.sat.i16.f32(float %f)
ret i16 %x
}
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 3a808b2f77441..15a4df2f698ad 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -5595,49 +5595,49 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s4
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s4
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
+; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v5.l, s1
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s1
; GFX11-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v4
-; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX11-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX11-TRUE16-NEXT: v_min_u32_e32 v7, 0xff, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 8, v4
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; GFX11-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, v1, v0
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v1, v0
; GFX11-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v6
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v7, v8
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, v7, v4
; GFX11-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v8
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, v3, v11
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v3, v7
; GFX11-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, v1, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[9:10], 24, v[7:8]
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, v1, v4
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: test_s_unsigned_v8f16_v8i8:
@@ -5707,52 +5707,51 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v2.l, s3
; GFX12-TRUE16-NEXT: s_lshr_b32 s3, s3, 16
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
+; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s4
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v4.l, s3
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v1.l, s2
-; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s1, 16
+; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v5.l, s1
+; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v3.l, s2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX12-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v6.l, s1
; GFX12-TRUE16-NEXT: v_min_u32_e32 v4, 0xff, v4
-; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
; GFX12-TRUE16-NEXT: v_min_u32_e32 v3, 0xff, v3
+; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 8, v0
; GFX12-TRUE16-NEXT: v_min_u32_e32 v7, 0xff, v2
; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v8, 8, v4
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; GFX12-TRUE16-NEXT: v_min_u32_e32 v5, 0xff, v5
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v4, v1, v0
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, v1, v0
; GFX12-TRUE16-NEXT: v_min_u32_e32 v1, 0xff, v6
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v7, v8
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, v7, v4
; GFX12-TRUE16-NEXT: v_cvt_u16_f16_e32 v0.l, s0
; GFX12-TRUE16-NEXT: v_or_b32_e32 v2, v5, v3
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff, v8
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v6
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v6
; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v2
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, v3, v11
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v3, v7
; GFX12-TRUE16-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v5, v1, v7
-; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[9:10], 24, v[7:8]
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v9, v1, v4
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v7
+; GFX12-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[4:5]
; GFX12-TRUE16-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v8
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v11
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v9.l
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GI-LABEL: test_s_unsigned_v8f16_v8i8:
@@ -6464,23 +6463,52 @@ define <4 x i16> @test_s_unsigned_v4f32_v4i16(<4 x float> inreg %f) {
; GFX9-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_s_unsigned_v4f32_v4i16:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
-; GFX11-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-FAKE16-LABEL: test_s_unsigned_v4f32_v4i16:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
+; GFX11-FAKE16-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX12-LABEL: test_s_unsigned_v4f32_v4i16:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
-; GFX12-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
-; GFX12-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_s_unsigned_v4f32_v4i16:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_pk_u16_f32 v0, s0, s0
+; GFX11-TRUE16-NEXT: v_cvt_pk_u16_f32 v1, s2, s2
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-FAKE16-LABEL: test_s_unsigned_v4f32_v4i16:
+; GFX12-FAKE16: ; %bb.0:
+; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
+; GFX12-FAKE16-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
+; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-TRUE16-LABEL: test_s_unsigned_v4f32_v4i16:
+; GFX12-TRUE16: ; %bb.0:
+; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_cvt_pk_u16_f32 v0, s0, s0
+; GFX12-TRUE16-NEXT: v_cvt_pk_u16_f32 v1, s2, s2
+; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-GI-LABEL: test_s_unsigned_v4f32_v4i16:
+; GFX12-GI: ; %bb.0:
+; GFX12-GI-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GI-NEXT: s_wait_expcnt 0x0
+; GFX12-GI-NEXT: s_wait_samplecnt 0x0
+; GFX12-GI-NEXT: s_wait_bvhcnt 0x0
+; GFX12-GI-NEXT: s_wait_kmcnt 0x0
+; GFX12-GI-NEXT: v_cvt_pk_u16_f32 v0, s0, s1
+; GFX12-GI-NEXT: v_cvt_pk_u16_f32 v1, s2, s3
+; GFX12-GI-NEXT: s_setpc_b64 s[30:31]
%x = call <4 x i16> @llvm.fptoui.sat.v4f32.v4i16(<4 x float> %f)
ret <4 x i16> %x
}
diff --git a/llvm/test/CodeGen/AMDGPU/function-returns.ll b/llvm/test/CodeGen/AMDGPU/function-returns.ll
index 019e49a823aa2..d0aaa756048f4 100644
--- a/llvm/test/CodeGen/AMDGPU/function-returns.ll
+++ b/llvm/test/CodeGen/AMDGPU/function-returns.ll
@@ -1113,24 +1113,26 @@ define <16 x i8> @v16i8_func_void() #0 {
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_load_b128 v[16:19], off, s[0:3], 0
+; GFX11-TRUE16-NEXT: buffer_load_b128 v[0:3], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v18.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v16i8_func_void:
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index 25d1f81ef8c1d..eefd4e74b6be0 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -3707,14 +3707,13 @@ define amdgpu_gfx void @test_call_external_void_func_v5i8() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v5i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v5i8 at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b64 v[3:4], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[5:6], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
@@ -3896,16 +3895,16 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v8i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v8i8 at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b64 v[8:9], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
@@ -4155,45 +4154,50 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8() #0 {
; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 0
; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v4, 16 :: v_dual_mov_b32 v5, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v32i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v32i8 at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b128 v[32:35], v[0:1], off
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 16 :: v_dual_mov_b32 v1, 0
-; GFX11-TRUE16-NEXT: global_load_b128 v[36:39], v[0:1], off
+; GFX11-TRUE16-NEXT: s_clause 0x1
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v[4:5], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v39
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v39.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v34.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
@@ -5236,17 +5240,16 @@ define amdgpu_gfx void @test_call_external_void_func_v5i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v5i8_ret at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v5i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b64 v[3:4], v[40:41], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[5:6], v[40:41], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[5:6], 24, v[3:4]
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v5.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[5:6]
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v5
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v6.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v0, v1, 0xc0c0004
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_perm_b32 v2, v2, v3, 0xc0c0004
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, 4 :: v_dual_mov_b32 v1, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v42, 0
@@ -5494,18 +5497,18 @@ define amdgpu_gfx void @test_call_external_void_func_v8i8_ret() #0 {
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v40, 0 :: v_dual_mov_b32 v41, 0
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v8i8_ret at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v8i8_ret at abs32@lo
-; GFX11-TRUE16-NEXT: global_load_b64 v[8:9], v[40:41], off
+; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[40:41], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v9.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v8.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
@@ -5877,42 +5880,46 @@ define amdgpu_gfx void @test_call_external_void_func_v32i8_ret() #0 {
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v3i8_ret at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v3i8_ret at abs32@lo
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: global_load_b128 v[32:35], v[40:41], off
-; GFX11-TRUE16-NEXT: global_load_b128 v[36:39], v[42:43], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v[40:41], off
+; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v[42:43], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v32
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v33
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v34
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v35
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v35
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v35, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v36, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v37, 24, v0
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 8, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 16, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 24, v36
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v37
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v38
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v39
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v39
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v35.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v16.l, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v37.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v38.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v39.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 8, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v33, 16, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 24, v16
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v22, 16, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v17
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v25, 8, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 24, v18
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v19
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v19
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v24.l, v18.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v19.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v35.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v36.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v37.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v32.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v33.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v34.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_perm_b32 v12, v12, v13, 0xc0c0004
; GFX11-TRUE16-NEXT: v_perm_b32 v13, v14, v15, 0xc0c0004
@@ -9603,26 +9610,28 @@ define amdgpu_gfx void @test_call_external_void_func_v16i8() #0 {
; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b128 v[16:19], v0, s[0:1]
+; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v0, s[0:1]
; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_v16i8 at abs32@hi
; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_v16i8 at abs32@lo
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v16
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v17
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v18
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v19
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v17.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v19.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v17, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v18, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v2
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v1.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v3.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v16.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v17.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v18.l
; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 1
@@ -17955,32 +17964,63 @@ define amdgpu_gfx void @test_call_external_void_func_bf16_inreg(i16 inreg %arg)
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: test_call_external_void_func_bf16_inreg:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s0, s33
-; GFX11-NEXT: s_mov_b32 s33, s32
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: v_writelane_b32 v40, s0, 2
-; GFX11-NEXT: s_add_i32 s32, s32, 16
-; GFX11-NEXT: v_writelane_b32 v40, s30, 0
-; GFX11-NEXT: v_writelane_b32 v40, s31, 1
-; GFX11-NEXT: s_mov_b32 s1, external_void_func_bf16 at abs32@hi
-; GFX11-NEXT: s_mov_b32 s0, external_void_func_bf16 at abs32@lo
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
-; GFX11-NEXT: v_readlane_b32 s30, v40, 0
-; GFX11-NEXT: v_readlane_b32 s31, v40, 1
-; GFX11-NEXT: s_mov_b32 s32, s33
-; GFX11-NEXT: v_readlane_b32 s0, v40, 2
-; GFX11-NEXT: s_or_saveexec_b32 s1, -1
-; GFX11-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
-; GFX11-NEXT: s_mov_b32 exec_lo, s1
-; GFX11-NEXT: s_mov_b32 s33, s0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: test_call_external_void_func_bf16_inreg:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, s33
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s32
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s0, 3
+; GFX11-TRUE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s4, 0
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s30, 1
+; GFX11-TRUE16-NEXT: v_writelane_b32 v40, s31, 2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-TRUE16-NEXT: s_mov_b32 s1, external_void_func_bf16 at abs32@hi
+; GFX11-TRUE16-NEXT: s_mov_b32 s0, external_void_func_bf16 at abs32@lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s4, v0
+; GFX11-TRUE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-TRUE16-NEXT: v_readlane_b32 s30, v40, 1
+; GFX11-TRUE16-NEXT: v_readlane_b32 s31, v40, 2
+; GFX11-TRUE16-NEXT: v_readlane_b32 s4, v40, 0
+; GFX11-TRUE16-NEXT: s_mov_b32 s32, s33
+; GFX11-TRUE16-NEXT: v_readlane_b32 s0, v40, 3
+; GFX11-TRUE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-TRUE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-TRUE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-TRUE16-NEXT: s_mov_b32 s33, s0
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: test_call_external_void_func_bf16_inreg:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, s33
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s32
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_store_b32 off, v40, s33 ; 4-byte Folded Spill
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s0, 2
+; GFX11-FAKE16-NEXT: s_add_i32 s32, s32, 16
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s30, 0
+; GFX11-FAKE16-NEXT: v_writelane_b32 v40, s31, 1
+; GFX11-FAKE16-NEXT: s_mov_b32 s1, external_void_func_bf16 at abs32@hi
+; GFX11-FAKE16-NEXT: s_mov_b32 s0, external_void_func_bf16 at abs32@lo
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_swappc_b64 s[30:31], s[0:1]
+; GFX11-FAKE16-NEXT: v_readlane_b32 s30, v40, 0
+; GFX11-FAKE16-NEXT: v_readlane_b32 s31, v40, 1
+; GFX11-FAKE16-NEXT: s_mov_b32 s32, s33
+; GFX11-FAKE16-NEXT: v_readlane_b32 s0, v40, 2
+; GFX11-FAKE16-NEXT: s_or_saveexec_b32 s1, -1
+; GFX11-FAKE16-NEXT: scratch_load_b32 v40, off, s33 ; 4-byte Folded Reload
+; GFX11-FAKE16-NEXT: s_mov_b32 exec_lo, s1
+; GFX11-FAKE16-NEXT: s_mov_b32 s33, s0
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SCRATCH-LABEL: test_call_external_void_func_bf16_inreg:
; GFX10-SCRATCH: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
index ee52b70b953e2..38c209dafd3c2 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fadd.ll
@@ -13717,114 +13717,58 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB54_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -13869,106 +13813,54 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB54_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB54_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB54_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB54_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14262,119 +14154,61 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB55_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14421,110 +14255,56 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB55_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB55_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB55_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB55_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB55_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -14827,119 +14607,61 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB56_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -14987,110 +14709,56 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB56_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB56_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB56_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB56_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB56_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -15390,111 +15058,56 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX1250-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX12-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX12-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v6, v3
+; GFX12-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB57_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -15525,116 +15138,65 @@ define void @global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshlrev_b32_sdwa v4, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX942-NEXT: v_and_or_b32 v4, v5, v6, v4
; GFX942-NEXT: buffer_wbl2 sc1
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB57_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB57_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB57_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v3, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB57_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: global_atomic_cmpswap v4, v[0:1], v[4:5], off sc0
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
+; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB57_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v6, v3
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB57_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: v_add_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB57_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fadd_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -16830,33 +16392,32 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -16864,7 +16425,7 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -16949,41 +16510,40 @@ define bfloat @global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB60_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_add_f32_e32 v3, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB60_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fadd_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -17627,121 +17187,62 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX1250-NEXT: v_lshrrev_b32_e32 v0, v3, v5
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
;
-; GFX12-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB62_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -17788,110 +17289,56 @@ define bfloat @global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB62_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB62_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB62_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB62_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB62_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fadd_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
index 3f674bbabca01..445191d16ec08 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmax.ll
@@ -8677,114 +8677,58 @@ define void @global_system_atomic_fmax_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -8829,106 +8773,54 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9180,119 +9072,61 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9339,110 +9173,56 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9702,119 +9482,61 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9850,122 +9572,68 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB38_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB38_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
+; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10225,111 +9893,56 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_neg__amdgpu_no_fine_
}
define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v6, v3
+; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_max_num_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10367,109 +9980,58 @@ define void @global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB39_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v3, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB39_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v6, v3
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: v_max_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmax_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11522,33 +11084,32 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v3, v3, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11556,7 +11117,7 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11641,41 +11202,40 @@ define bfloat @global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_max_f32_e32 v3, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmax_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12247,121 +11807,62 @@ define void @global_agent_atomic_fmax_noret_bf16__offset12b__align4_pos__amdgpu_
}
define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_max_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_max_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_max_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12408,110 +11909,56 @@ define bfloat @global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_max_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_max_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_max_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fmax_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
index 7c09a40e7c578..8b34afe64b1d0 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fmin.ll
@@ -8677,114 +8677,58 @@ define void @global_system_atomic_fmin_noret_f16__offset12b_pos__amdgpu_no_fine_
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB36_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -8829,106 +8773,54 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB36_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB36_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB36_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB36_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9180,119 +9072,61 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__amdgpu_no_fine_grained_memory(
}
define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB37_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9339,110 +9173,56 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB37_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB37_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB37_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB37_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -9702,119 +9482,61 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_
}
define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB38_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -9850,122 +9572,68 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
; GFX942-NEXT: buffer_wbl2 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB38_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB38_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
+; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB38_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB38_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB38_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB38_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -10225,111 +9893,56 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_neg__amdgpu_no_fine_
}
define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX12-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX12-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v6, v3
+; GFX12-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_min_num_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB39_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -10367,109 +9980,58 @@ define void @global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory(
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v4, v5
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB39_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB39_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v3, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB39_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB39_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v6, v3
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB39_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: v_min_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB39_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fmin_noret_bf16__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
@@ -11522,33 +11084,32 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v3, v3, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -11556,7 +11117,7 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -11641,41 +11202,40 @@ define bfloat @global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_
; GFX11-TRUE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB42_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_min_f32_e32 v3, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB42_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fmin_ret_bf16__offset12b_pos__align4__amdgpu_no_fine_grained_memory:
@@ -12247,121 +11807,62 @@ define void @global_agent_atomic_fmin_noret_bf16__offset12b__align4_pos__amdgpu_
}
define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_min_num_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_min_num_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_min_num_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB44_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX942: ; %bb.0:
@@ -12408,110 +11909,56 @@ define bfloat @global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_min_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB44_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_min_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB44_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB44_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_min_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB44_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fmin_ret_bf16__offset12b_pos__amdgpu_no_fine_grained_memory:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
index a42ff751eac7d..42a8596598bce 100644
--- a/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-atomicrmw-fsub.ll
@@ -9223,114 +9223,58 @@ define void @global_system_atomic_fsub_noret_f16__offset12b_pos(ptr addrspace(1)
; --------------------------------------------------------------------
define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB32_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16:
; GFX942: ; %bb.0:
@@ -9375,106 +9319,54 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v3, v4
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB32_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB32_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB32_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB32_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16:
; GFX10: ; %bb.0:
@@ -9724,119 +9616,61 @@ define bfloat @global_agent_atomic_fsub_ret_bf16(ptr addrspace(1) %ptr, bfloat %
}
define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB33_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -9883,110 +9717,56 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB33_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB33_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB33_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB33_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
@@ -10244,119 +10024,61 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1)
}
define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB34_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX942: ; %bb.0:
@@ -10393,121 +10115,67 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: global_atomic_cmpswap v2, v[0:1], v[2:3], off sc0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: buffer_inv sc1
-; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB34_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0xfffff800, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: buffer_inv sc1
+; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
+; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_cbranch_execnz .LBB34_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
+; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB34_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB34_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0xfffff800, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB34_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB34_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_neg:
; GFX10: ; %bb.0:
@@ -10765,111 +10433,56 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_neg(ptr addrspace(1)
}
define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX12-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX12-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX12-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX12-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: global_load_b32 v4, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v6, v3
+; GFX12-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX12-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX12-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX12-NEXT: v_mov_b32_e32 v4, v3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB35_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_agent_atomic_fsub_noret_bf16:
; GFX942: ; %bb.0:
@@ -10908,108 +10521,57 @@ define void @global_agent_atomic_fsub_noret_bf16(ptr addrspace(1) %ptr, bfloat %
; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX942-NEXT: v_mov_b32_e32 v5, v4
; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB35_1
-; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, v0
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v5.l
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v2, v4, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v5, v2
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, v4, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v6
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX11-TRUE16-NEXT: v_and_or_b32 v2, v3, v5, v2
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[2:3], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, v2
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_noret_bf16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 3, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v6, v3
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB35_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, v5, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v3, v3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v3, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v3
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_and_or_b32 v3, v4, v6, v3
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, v3
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB35_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX942-NEXT: s_cbranch_execnz .LBB35_1
+; GFX942-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: global_agent_atomic_fsub_noret_bf16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_lshlrev_b32 v2, 16, v2
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v3
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v5, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v6, v3
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB35_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: v_sub_f32_e32 v3, v3, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_bfe_u32 v7, v3, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v3
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-NEXT: v_add3_u32 v7, v7, v3, 0x7fff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc_lo
+; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, v5, v3
+; GFX11-NEXT: v_and_or_b32 v3, v4, v6, v3
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
+; GFX11-NEXT: v_mov_b32_e32 v4, v3
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB35_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_agent_atomic_fsub_noret_bf16:
; GFX10: ; %bb.0:
@@ -12056,33 +11618,32 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX12-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX12-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX12-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX12-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX12-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX12-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
+; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -12090,7 +11651,7 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -12175,41 +11736,40 @@ define bfloat @global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4(ptr addr
; GFX11-TRUE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: global_load_b32 v2, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off offset:2046
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: .p2align 6
; GFX11-TRUE16-NEXT: .LBB38_1: ; %atomicrmw.start
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v2, v2, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX11-TRUE16-NEXT: v_sub_f32_e32 v3, v3, v2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v2, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
-; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v2, 0x7fff
+; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v3, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v6, 0x400000, v3
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
+; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v3, 0x7fff
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc_lo
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v4, 0xffff0000, v5, v2
+; GFX11-TRUE16-NEXT: v_and_or_b32 v3, 0xffff0000, v4, v3
; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], v[4:5], off offset:2046 glc
+; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v3, v[0:1], v[3:4], off offset:2046 glc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: buffer_gl1_inv
; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v5
+; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v3, v4
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB38_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.l
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: global_agent_atomic_fsub_ret_bf16__offset12b_pos__align4:
@@ -12777,121 +12337,62 @@ define void @global_agent_atomic_fsub_noret_bf16__offset12b__align4_pos(ptr addr
}
define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1) %ptr, bfloat %val) #0 {
-; GFX12-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_expcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX12-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX12-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX12-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX12-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX12-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX12-TRUE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_storecnt 0x0
-; GFX12-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_expcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_samplecnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_bvhcnt 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX12-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX12-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX12-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX12-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX12-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX12-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX12-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX12-FAKE16-NEXT: global_wb scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_storecnt 0x0
-; GFX12-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: global_inv scope:SCOPE_SYS
-; GFX12-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
-; GFX12-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX12-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX12-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX12-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX12-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX12-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: global_load_b32 v5, v[0:1], off
+; GFX12-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX12-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_not_b32_e32 v4, v4
+; GFX12-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v6, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX12-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX12-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX12-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX12-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX12-NEXT: global_wb scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_storecnt 0x0
+; GFX12-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off th:TH_ATOMIC_RETURN scope:SCOPE_SYS
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: global_inv scope:SCOPE_SYS
+; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: s_cbranch_execnz .LBB40_1
+; GFX12-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX12-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX12-NEXT: s_setpc_b64 s[30:31]
;
; GFX942-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX942: ; %bb.0:
@@ -12938,110 +12439,56 @@ define bfloat @global_system_atomic_fsub_ret_bf16__offset12b_pos(ptr addrspace(1
; GFX942-NEXT: v_lshrrev_b32_e32 v0, v4, v2
; GFX942-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_add_co_u32 v2, vcc_lo, 0x7fe, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v3.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, -4, v2
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
-; GFX11-TRUE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_not_b32_e32 v3, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-TRUE16-NEXT: .p2align 6
-; GFX11-TRUE16-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, v2, v6
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_sub_f32_e32 v5, v5, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, v2, v5
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_and_or_b32 v5, v6, v3, v5
-; GFX11-TRUE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-TRUE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: buffer_gl1_inv
-; GFX11-TRUE16-NEXT: buffer_gl0_inv
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-TRUE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, v2, v5
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, -4, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_mov_b32 s0, 0
-; GFX11-FAKE16-NEXT: global_load_b32 v5, v[0:1], off
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
-; GFX11-FAKE16-NEXT: v_not_b32_e32 v4, v4
-; GFX11-FAKE16-NEXT: .p2align 6
-; GFX11-FAKE16-NEXT: .LBB40_1: ; %atomicrmw.start
-; GFX11-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v6, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_sub_f32_e32 v5, v5, v2
-; GFX11-FAKE16-NEXT: v_bfe_u32 v7, v5, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v5
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_or_b32 v5, v6, v4, v5
-; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-FAKE16-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: buffer_gl1_inv
-; GFX11-FAKE16-NEXT: buffer_gl0_inv
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
-; GFX11-FAKE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-FAKE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: s_cbranch_execnz .LBB40_1
-; GFX11-FAKE16-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-FAKE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, v3, v5
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_add_co_u32 v3, vcc_lo, 0x7fe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_and_b32_e32 v0, -4, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: global_load_b32 v5, v[0:1], off
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v4, v4
+; GFX11-NEXT: .p2align 6
+; GFX11-NEXT: .LBB40_1: ; %atomicrmw.start
+; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v6, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, v3, v6
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e32 v5, v5, v2
+; GFX11-NEXT: v_bfe_u32 v7, v5, 16, 1
+; GFX11-NEXT: v_or_b32_e32 v8, 0x400000, v5
+; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_add3_u32 v7, v7, v5, 0x7fff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v7, v8, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 16, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, v3, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v5, v6, v4, v5
+; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT: global_atomic_cmpswap_b32 v5, v[0:1], v[5:6], off glc
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: buffer_gl1_inv
+; GFX11-NEXT: buffer_gl0_inv
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, v5, v6
+; GFX11-NEXT: s_or_b32 s0, vcc_lo, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: s_cbranch_execnz .LBB40_1
+; GFX11-NEXT: ; %bb.2: ; %atomicrmw.end
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, v3, v5
+; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: global_system_atomic_fsub_ret_bf16__offset12b_pos:
; GFX10: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
index a3ddbdcdcdd50..6930799100a9a 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-cc.ll
@@ -559,10 +559,12 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_half(half inreg %a, half %b) {
; DAGISEL-GFX11-NEXT: {{ $}}
; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY3]]
; DAGISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY]]
; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
@@ -627,10 +629,12 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_bfloat(bfloat inreg %a, bfloat %
; DAGISEL-GFX11-NEXT: {{ $}}
; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY3]]
; DAGISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY]]
; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
@@ -695,10 +699,12 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_cc_i16(i16 inreg %a, i16 %b) {
; DAGISEL-GFX11-NEXT: {{ $}}
; DAGISEL-GFX11-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr8
; DAGISEL-GFX11-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
+ ; DAGISEL-GFX11-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY2]].lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKUP 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
; DAGISEL-GFX11-NEXT: [[SI_PC_ADD_REL_OFFSET:%[0-9]+]]:sreg_64 = SI_PC_ADD_REL_OFFSET target-flags(amdgpu-gotprel32-lo) @use, target-flags(amdgpu-gotprel32-hi) @use, implicit-def dead $scc
; DAGISEL-GFX11-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM killed [[SI_PC_ADD_REL_OFFSET]], 0, 0 :: (dereferenceable invariant load (s64) from got, addrspace 4)
- ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY1]]
+ ; DAGISEL-GFX11-NEXT: $vgpr0_lo16 = COPY [[COPY3]]
; DAGISEL-GFX11-NEXT: $vgpr1_lo16 = COPY [[COPY]]
; DAGISEL-GFX11-NEXT: $sgpr30_sgpr31 = SI_CALL killed [[S_LOAD_DWORDX2_IMM]], @use, csr_amdgpu_si_gfx, implicit $vgpr0_lo16, implicit $vgpr1_lo16
; DAGISEL-GFX11-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def dead $scc, implicit-def $sgpr32, implicit $sgpr32
diff --git a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
index f51d7f5a2fff8..8f895d5a4cce8 100644
--- a/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/isel-amdgpu-cs-chain-preserve-cc.ll
@@ -739,13 +739,12 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_half(half inre
; GISEL-GFX11-TRUE16-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-TRUE16-NEXT: {{ $}}
; GISEL-GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY]]
- ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY3]], 0, [[COPY2]], 0, 0, 0, implicit $mode, implicit $exec
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 [[COPY4]], [[V_ADD_F16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into `ptr poison`)
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]]
+ ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_ADD_F16_t16_e64 0, [[COPY2]], 0, [[COPY1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 [[COPY3]], [[V_ADD_F16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (f16) into `ptr poison`)
; GISEL-GFX11-TRUE16-NEXT: S_ENDPGM 0
;
; GISEL-GFX11-FAKE16-LABEL: name: amdgpu_cs_chain_preserve_cc_half
@@ -947,13 +946,12 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_cc_i16(i16 inreg
; GISEL-GFX11-TRUE16-NEXT: liveins: $sgpr0, $vgpr8
; GISEL-GFX11-TRUE16-NEXT: {{ $}}
; GISEL-GFX11-TRUE16-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr0
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr8
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY1]].lo16
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY $vgpr8
; GISEL-GFX11-TRUE16-NEXT: [[DEF:%[0-9]+]]:sreg_64 = IMPLICIT_DEF
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vgpr_16 = COPY [[COPY]]
- ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_NC_U16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ADD_NC_U16_t16_e64 0, [[COPY3]], 0, [[COPY2]], 0, 0, implicit $exec
- ; GISEL-GFX11-TRUE16-NEXT: [[COPY4:%[0-9]+]]:vreg_64 = COPY [[DEF]]
- ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 [[COPY4]], [[V_ADD_NC_U16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i16) into `ptr poison`)
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY2:%[0-9]+]]:vgpr_16 = COPY [[COPY]]
+ ; GISEL-GFX11-TRUE16-NEXT: [[V_ADD_NC_U16_t16_e64_:%[0-9]+]]:vgpr_16 = V_ADD_NC_U16_t16_e64 0, [[COPY2]], 0, [[COPY1]], 0, 0, implicit $exec
+ ; GISEL-GFX11-TRUE16-NEXT: [[COPY3:%[0-9]+]]:vreg_64 = COPY [[DEF]]
+ ; GISEL-GFX11-TRUE16-NEXT: FLAT_STORE_SHORT_t16 [[COPY3]], [[V_ADD_NC_U16_t16_e64_]], 0, 0, implicit $exec, implicit $flat_scr :: (store (i16) into `ptr poison`)
; GISEL-GFX11-TRUE16-NEXT: S_ENDPGM 0
;
; GISEL-GFX11-FAKE16-LABEL: name: amdgpu_cs_chain_preserve_cc_i16
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
index cd41c21ac8862..945ef4bb560ec 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.bitop3.ll
@@ -326,11 +326,8 @@ define amdgpu_ps half @bitop3_b16_sss(i16 inreg %a, i16 inreg %b, i16 inreg %c)
; GFX1250-GISEL-TRUE16-NEXT: v_nop
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, s1, v0.l bitop3:0x12
-; GFX1250-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, s0, s1, v0.l bitop3:0x12
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
%ret = call i16 @llvm.amdgcn.bitop3.i16(i16 %a, i16 %b, i16 %c, i32 18)
%ret_cast = bitcast i16 %ret to half
@@ -393,9 +390,9 @@ define amdgpu_ps half @bitop3_b16_vii(i16 %a) {
; GFX1250-SDG-TRUE16-NEXT: s_mov_b64 s[64:65], 0
; GFX1250-SDG-TRUE16-NEXT: v_nop
; GFX1250-SDG-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0x7d0
+; GFX1250-SDG-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x7d0
; GFX1250-SDG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v1.l, 0x3e8 bitop3:0x14
+; GFX1250-SDG-TRUE16-NEXT: v_bitop3_b16 v0.l, v0.l, v0.h, 0x3e8 bitop3:0x14
; GFX1250-SDG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDG-FAKE16-LABEL: bitop3_b16_vii:
@@ -485,11 +482,8 @@ define amdgpu_ps half @bitop3_b16_iii() {
; GFX1250-GISEL-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0x7d0
; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0x3e8
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, 0xbb8, v0.l, v0.h bitop3:0x15
-; GFX1250-GISEL-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX1250-GISEL-TRUE16-NEXT: v_bitop3_b16 v0.l, 0xbb8, v0.l, v0.h bitop3:0x15
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: bitop3_b16_iii:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
index 5f51c4c1c5e6f..57e46ef545e50 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.bf16.ll
@@ -17,13 +17,13 @@ define bfloat @v_exp2_bf16(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
@@ -565,13 +565,13 @@ define bfloat @v_exp2_bf16_fast(bfloat %in) {
; GFX1200-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0xc2fc0000, v0
; GFX1200-SDAG-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v2, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42800000, vcc_lo
+; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0xffffffc0, vcc_lo
-; GFX1200-SDAG-TRUE16-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
+; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-SDAG-TRUE16-NEXT: v_exp_f32_e32 v0, v0
; GFX1200-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
; GFX1200-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index 2bcb08745c1e4..25bc0a87a0c32 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -3821,20 +3821,35 @@ define float @v_log2_f32_from_fpext_bf16(bfloat %src) {
; GFX900-GISEL-NEXT: v_sub_f32_e32 v0, v0, v1
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX1100-SDAG-LABEL: v_log2_f32_from_fpext_bf16:
-; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v0, v0, v2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1100-SDAG-NEXT: v_log_f32_e32 v0, v0
-; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1100-SDAG-NEXT: s_setpc_b64 s[30:31]
+; GFX1100-SDAG-TRUE16-LABEL: v_log2_f32_from_fpext_bf16:
+; GFX1100-SDAG-TRUE16: ; %bb.0:
+; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1100-SDAG-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 32, vcc_lo
+; GFX1100-SDAG-TRUE16-NEXT: v_ldexp_f32 v0, v0, v1
+; GFX1100-SDAG-TRUE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
+; GFX1100-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1100-SDAG-TRUE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-SDAG-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX1100-SDAG-TRUE16-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX1100-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX1100-SDAG-FAKE16-LABEL: v_log2_f32_from_fpext_bf16:
+; GFX1100-SDAG-FAKE16: ; %bb.0:
+; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-SDAG-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX1100-SDAG-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0x800000, v0
+; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v2, 0, 32, vcc_lo
+; GFX1100-SDAG-FAKE16-NEXT: v_cndmask_b32_e64 v1, 0, 0x42000000, vcc_lo
+; GFX1100-SDAG-FAKE16-NEXT: v_ldexp_f32 v0, v0, v2
+; GFX1100-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1100-SDAG-FAKE16-NEXT: v_log_f32_e32 v0, v0
+; GFX1100-SDAG-FAKE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX1100-SDAG-FAKE16-NEXT: v_sub_f32_e32 v0, v0, v1
+; GFX1100-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX1100-GISEL-LABEL: v_log2_f32_from_fpext_bf16:
; GFX1100-GISEL: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.powi.ll b/llvm/test/CodeGen/AMDGPU/llvm.powi.ll
index fd5ab9296cb7c..a8e2e841128b2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.powi.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.powi.ll
@@ -89,20 +89,19 @@ define i16 @v_powi_f16(i16 %l, i32 %r) {
; GFX11-TRUE16-LABEL: v_powi_f16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.l
-; GFX11-TRUE16-NEXT: v_trunc_f32_e32 v6, v1
+; GFX11-TRUE16-NEXT: v_cvt_f32_i32_e32 v1, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v0.h
+; GFX11-TRUE16-NEXT: v_trunc_f32_e32 v7, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e64 s0, 0x800000, |v2|
; GFX11-TRUE16-NEXT: v_cmp_class_f32_e64 s2, v2, 24
-; GFX11-TRUE16-NEXT: v_cmp_neq_f32_e64 s1, v6, v1
+; GFX11-TRUE16-NEXT: v_cmp_neq_f32_e64 s1, v7, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v4, 0, 32, s0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v3, 0, 0x42000000, s0
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, v6, v1
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, v7, v1
; GFX11-TRUE16-NEXT: v_ldexp_f32 v4, |v2|, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_log_f32_e32 v4, v4
@@ -116,9 +115,9 @@ define i16 @v_powi_f16(i16 %l, i32 %r) {
; GFX11-TRUE16-NEXT: v_dual_add_f32 v3, v3, v4 :: v_dual_mul_f32 v4, 0.5, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_exp_f32_e32 v3, v3
-; GFX11-TRUE16-NEXT: v_trunc_f32_e32 v7, v4
+; GFX11-TRUE16-NEXT: v_trunc_f32_e32 v6, v4
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_cmp_lg_f32_e32 vcc_lo, v7, v4
+; GFX11-TRUE16-NEXT: v_cmp_lg_f32_e32 vcc_lo, v6, v4
; GFX11-TRUE16-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-TRUE16-NEXT: v_ldexp_f32 v3, v3, v5
; GFX11-TRUE16-NEXT: s_and_b32 vcc_lo, s0, vcc_lo
diff --git a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
index c90d9909ddc4f..b04cf6b4e4d4c 100644
--- a/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/maximumnum.bf16.ll
@@ -107,11 +107,8 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
@@ -162,11 +159,8 @@ define bfloat @v_maximumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
@@ -293,19 +287,16 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -332,20 +323,17 @@ define bfloat @v_maximumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -12542,11 +12530,8 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_maximumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
@@ -12597,11 +12582,8 @@ define bfloat @v_maximumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
diff --git a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
index 7615f660c98b5..6289e433e26d5 100644
--- a/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/minimumnum.bf16.ll
@@ -109,11 +109,8 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
@@ -164,11 +161,8 @@ define bfloat @v_minimumnum_bf16(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
@@ -298,19 +292,16 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_nnan:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX11-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX11-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -337,20 +328,17 @@ define bfloat @v_minimumnum_bf16_nnan(bfloat %x, bfloat %y) #1 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v1.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e64 s0, 0x8000, v0.l
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-TRUE16-NEXT: v_cmp_lt_f32_e32 vcc_lo, v3, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.l, v0.l, vcc_lo
-; GFX12-TRUE16-NEXT: v_cmp_eq_u16_e32 vcc_lo, 0x8000, v0.l
; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e64 s0, 0, v2
-; GFX12-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX12-TRUE16-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v2
+; GFX12-TRUE16-NEXT: s_and_b32 s0, vcc_lo, s0
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: v_cndmask_b16 v0.l, v1.l, v0.l, s0
; GFX12-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -12577,11 +12565,8 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX11-TRUE16-LABEL: v_minimumnum_bf16_no_ieee:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
@@ -12632,11 +12617,8 @@ define bfloat @v_minimumnum_bf16_no_ieee(bfloat %x, bfloat %y) #0 {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GFX12-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX12-TRUE16-NEXT: v_cmp_u_f32_e64 s0, v3, v3
diff --git a/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll b/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
index 119024a395946..e07f8e5fce396 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar-float-sop1.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel=0 -mtriple=amdgpu11.50 < %s | FileCheck %s --check-prefixes=CHECK,SDAG
+; RUN: llc -global-isel=0 -mtriple=amdgpu11.50 < %s | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-GFX11
; RUN: llc -global-isel=1 -mtriple=amdgpu11.50 < %s | FileCheck %s --check-prefixes=CHECK,GISEL-GFX11
-; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 < %s | FileCheck %s --check-prefixes=CHECK,SDAG
+; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 < %s | FileCheck %s --check-prefixes=CHECK,SDAG,SDAG-GFX12
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 < %s | FileCheck %s --check-prefixes=CHECK,GISEL-GFX12
define amdgpu_vs float @sitofp_i32_to_f32(i32 inreg %val) {
@@ -45,19 +45,63 @@ define amdgpu_vs i32 @fptoui_f32_to_u32(float inreg %val) {
}
define amdgpu_vs i16 @fptosi_f32_to_i16(float inreg %val) {
-; CHECK-LABEL: fptosi_f32_to_i16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_cvt_i32_f32 s0, s0
-; CHECK-NEXT: ; return to shader part epilog
+; SDAG-GFX11-LABEL: fptosi_f32_to_i16:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_cvt_i32_f32 s0, s0
+; SDAG-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: fptosi_f32_to_i16:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_cvt_i32_f32 s0, s0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; SDAG-GFX12-LABEL: fptosi_f32_to_i16:
+; SDAG-GFX12: ; %bb.0:
+; SDAG-GFX12-NEXT: s_cvt_i32_f32 s0, s0
+; SDAG-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX12-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; SDAG-GFX12-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX12-LABEL: fptosi_f32_to_i16:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_cvt_i32_f32 s0, s0
+; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = fptosi float %val to i16
ret i16 %res
}
define amdgpu_vs i16 @fptoui_f32_to_u16(float inreg %val) {
-; CHECK-LABEL: fptoui_f32_to_u16:
-; CHECK: ; %bb.0:
-; CHECK-NEXT: s_cvt_u32_f32 s0, s0
-; CHECK-NEXT: ; return to shader part epilog
+; SDAG-GFX11-LABEL: fptoui_f32_to_u16:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_cvt_u32_f32 s0, s0
+; SDAG-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-GFX11-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX11-LABEL: fptoui_f32_to_u16:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_cvt_u32_f32 s0, s0
+; GISEL-GFX11-NEXT: ; return to shader part epilog
+;
+; SDAG-GFX12-LABEL: fptoui_f32_to_u16:
+; SDAG-GFX12: ; %bb.0:
+; SDAG-GFX12-NEXT: s_cvt_u32_f32 s0, s0
+; SDAG-GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; SDAG-GFX12-NEXT: v_mov_b32_e32 v0, s0
+; SDAG-GFX12-NEXT: v_readfirstlane_b32 s0, v0
+; SDAG-GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
+; SDAG-GFX12-NEXT: ; return to shader part epilog
+;
+; GISEL-GFX12-LABEL: fptoui_f32_to_u16:
+; GISEL-GFX12: ; %bb.0:
+; GISEL-GFX12-NEXT: s_cvt_u32_f32 s0, s0
+; GISEL-GFX12-NEXT: ; return to shader part epilog
%res = fptoui float %val to i16
ret i16 %res
}
diff --git a/llvm/test/CodeGen/AMDGPU/usubsat.ll b/llvm/test/CodeGen/AMDGPU/usubsat.ll
index 8ea1c4fa0adc7..0dc5a60b2d066 100644
--- a/llvm/test/CodeGen/AMDGPU/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/usubsat.ll
@@ -47,15 +47,25 @@ define i8 @s_usubsat_i8(i8 inreg %lhs, i8 inreg %rhs) {
; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: s_usubsat_i8:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_max_u32 s0, s0, s1
-; GFX11-NEXT: s_sub_i32 s0, s0, s1
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_setpc_b64 s[30:31]
+; GFX11-TRUE16-LABEL: s_usubsat_i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_max_u32_e32 v1, v1, v0
+; GFX11-TRUE16-NEXT: v_sub_nc_u32_e32 v0, v1, v0
+; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-FAKE16-LABEL: s_usubsat_i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_max_u32 s0, s0, s1
+; GFX11-FAKE16-NEXT: s_sub_i32 s0, s0, s1
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%result = call i8 @llvm.usub.sat.i8(i8 %lhs, i8 %rhs)
ret i8 %result
}
More information about the llvm-branch-commits
mailing list